xnu-3789.70.16.tar.gz

[apple/xnu.git] / bsd / sys / mcache.h

diff --git a/bsd/sys/mcache.h b/bsd/sys/mcache.h
index 34c76988f935e2b6116a778726ac27167a5b6cc2..906363384dd925a8fe35c323bf2e1189573adbc0 100644 (file)
--- a/bsd/sys/mcache.h
+++ b/bsd/sys/mcache.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2006-2013 Apple Inc. All rights reserved.
+ * Copyright (c) 2006-2015 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
@@ -37,6 +37,7 @@ extern "C" {
 #include <sys/types.h>
 #include <sys/queue.h>
 #include <mach/boolean.h>
+#include <mach/branch_predicates.h>
 #include <kern/locks.h>
 #include <libkern/OSAtomic.h>
 
@@ -51,7 +52,8 @@ extern "C" {
 /*
  * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG build.
  */
-#define        VERIFY(EX)      ((void)((EX) || assfail(#EX, __FILE__, __LINE__)))
+#define        VERIFY(EX)      \
+       ((void)(__probable((EX)) || assfail(#EX, __FILE__, __LINE__)))
 #if DEBUG
 #define        ASSERT(EX)      VERIFY(EX)
 #else
@@ -61,8 +63,7 @@ extern "C" {
 /*
  * Compile time assert; this should be on its own someday.
  */
-#define        _CASSERT(x)     \
-       switch (0) { case 0: case (x): ; }
+#define        _CASSERT(x)     _Static_assert(x, "compile-time assertion failed")
 
 /*
  * Atomic macros; these should be on their own someday.
@@ -85,8 +86,19 @@ extern "C" {
 #define        atomic_add_64(a, n)                                             \
        ((void) atomic_add_64_ov(a, n))
 
+#define        atomic_test_set_32(a, o, n)                                     \
+       OSCompareAndSwap(o, n, (volatile UInt32 *)a)
+
+#define        atomic_set_32(a, n) do {                                        \
+       while (!atomic_test_set_32(a, *a, n))                           \
+               ;                                                       \
+} while (0)
+
+#define        atomic_test_set_64(a, o, n)                                     \
+       OSCompareAndSwap64(o, n, (volatile UInt64 *)a)
+
 #define        atomic_set_64(a, n) do {                                        \
-       while (!OSCompareAndSwap64(*a, n, (volatile UInt64 *)a))        \
+       while (!atomic_test_set_64(a, *a, n))                           \
                ;                                                       \
 } while (0)
 
@@ -100,6 +112,14 @@ extern "C" {
 } while (0)
 #endif /* __LP64__ */
 
+#define        atomic_test_set_ptr(a, o, n)                                    \
+       OSCompareAndSwapPtr(o, n, (void * volatile *)a)
+
+#define        atomic_set_ptr(a, n) do {                                       \
+       while (!atomic_test_set_ptr(a, *a, n))                          \
+               ;                                                       \
+} while (0)
+
 #define        atomic_or_8_ov(a, n)                                            \
        ((u_int8_t) OSBitOrAtomic8(n, (volatile UInt8 *)a))
 
@@ -154,11 +174,13 @@ extern "C" {
 #define        atomic_bitclear_32(a, n)                                        \
        atomic_and_32(a, ~(n))
 
+#define        membar_sync     OSMemoryBarrier
+
 /*
  * Use CPU_CACHE_LINE_SIZE instead of MAX_CPU_CACHE_LINE_SIZE, unless
  * wasting space is of no concern.
  */
-#define        MAX_CPU_CACHE_LINE_SIZE 64
+#define        MAX_CPU_CACHE_LINE_SIZE 128
 #define        CPU_CACHE_LINE_SIZE     mcache_cache_line_size()
 
 #ifndef IS_P2ALIGNED
@@ -244,7 +266,7 @@ typedef struct mcache_cpu {
        int             cc_objs;        /* number of objects in filled bkt */
        int             cc_pobjs;       /* number of objects in previous bkt */
        int             cc_bktsize;     /* number of elements in a full bkt */
-} __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE), packed)) mcache_cpu_t;
+} __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE))) mcache_cpu_t;
 
 typedef unsigned int (*mcache_allocfn_t)(void *, mcache_obj_t ***,
     unsigned int, int);
@@ -303,7 +325,8 @@ typedef struct mcache {
        /*
         * Per-CPU layer, aligned at cache line boundary
         */
-       mcache_cpu_t    mc_cpu[1];
+       mcache_cpu_t    mc_cpu[1]
+           __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE)));
 } mcache_t;
 
 #define        MCACHE_ALIGN    8       /* default guaranteed alignment */
@@ -324,22 +347,23 @@ typedef struct mcache {
 
 #define        MCACHE_STACK_DEPTH 16
 
+#define        MCA_TRN_MAX     2               /* Number of transactions to record */
+
 typedef struct mcache_audit {
        struct mcache_audit *mca_next;  /* next audit struct */
        void            *mca_addr;      /* address of buffer */
        mcache_t        *mca_cache;     /* parent cache of the buffer */
-       struct thread   *mca_thread;    /* thread doing transaction */
-       struct thread   *mca_pthread;   /* previous transaction thread */
        size_t          mca_contents_size; /* size of saved contents */
        void            *mca_contents;  /* user-specific saved contents */
-       uint32_t        mca_tstamp;     /* transaction timestamp (ms) */
-       uint32_t        mca_ptstamp;    /* prev transaction timestamp (ms) */
-       uint16_t        mca_depth;      /* pc stack depth */
-       uint16_t        mca_pdepth;     /* previous transaction pc stack */
-       void            *mca_stack[MCACHE_STACK_DEPTH];
-       void            *mca_pstack[MCACHE_STACK_DEPTH];
        void            *mca_uptr;      /* user-specific pointer */
        uint32_t        mca_uflags;     /* user-specific flags */
+       uint32_t        mca_next_trn;
+       struct mca_trn {
+               struct thread   *mca_thread;    /* thread doing transaction */
+               uint32_t        mca_tstamp;
+               uint16_t        mca_depth;
+               void            *mca_stack[MCACHE_STACK_DEPTH];
+       } mca_trns[MCA_TRN_MAX];
 } mcache_audit_t;
 
 __private_extern__ int assfail(const char *, const char *, int);
@@ -358,7 +382,7 @@ __private_extern__ unsigned int mcache_alloc_ext(mcache_t *, mcache_obj_t **,
     unsigned int, int);
 __private_extern__ void mcache_free_ext(mcache_t *, mcache_obj_t *);
 __private_extern__ void mcache_reap(void);
-__private_extern__ boolean_t mcache_purge_cache(mcache_t *);
+__private_extern__ boolean_t mcache_purge_cache(mcache_t *, boolean_t);
 __private_extern__ void mcache_waiter_inc(mcache_t *);
 __private_extern__ void mcache_waiter_dec(mcache_t *);
 __private_extern__ boolean_t mcache_bkt_isempty(mcache_t *);
@@ -377,6 +401,10 @@ __private_extern__ char *mcache_dump_mca(mcache_audit_t *);
 __private_extern__ void mcache_audit_panic(mcache_audit_t *, void *, size_t,
     int64_t, int64_t);
 
+extern int32_t total_sbmb_cnt;
+extern int32_t total_sbmb_cnt_floor;
+extern int32_t total_sbmb_cnt_peak;
+extern int64_t sbmb_limreached;
 extern mcache_t *mcache_audit_cache;
 
 #ifdef  __cplusplus