X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/39236c6e673c41db228275375ab7fdb0f837b292..4d15aeb193b2c68f1d38666c317f8d3734f5f083:/bsd/sys/mcache.h

diff --git a/bsd/sys/mcache.h b/bsd/sys/mcache.h
index 34c76988f..906363384 100644
--- a/bsd/sys/mcache.h
+++ b/bsd/sys/mcache.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2006-2013 Apple Inc. All rights reserved.
+ * Copyright (c) 2006-2015 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
@@ -37,6 +37,7 @@ extern "C" {
 #include <sys/types.h>
 #include <sys/queue.h>
 #include <mach/boolean.h>
+#include <mach/branch_predicates.h>
 #include <kern/locks.h>
 #include <libkern/OSAtomic.h>
 
@@ -51,7 +52,8 @@ extern "C" {
 /*
  * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG build.
  */
-#define	VERIFY(EX)	((void)((EX) || assfail(#EX, __FILE__, __LINE__)))
+#define	VERIFY(EX)	\
+	((void)(__probable((EX)) || assfail(#EX, __FILE__, __LINE__)))
 #if DEBUG
 #define	ASSERT(EX)	VERIFY(EX)
 #else
@@ -61,8 +63,7 @@ extern "C" {
 /*
  * Compile time assert; this should be on its own someday.
  */
-#define	_CASSERT(x)	\
-	switch (0) { case 0: case (x): ; }
+#define	_CASSERT(x)	_Static_assert(x, "compile-time assertion failed")
 
 /*
  * Atomic macros; these should be on their own someday.
@@ -85,8 +86,19 @@ extern "C" {
 #define	atomic_add_64(a, n)						\
 	((void) atomic_add_64_ov(a, n))
 
+#define	atomic_test_set_32(a, o, n)					\
+	OSCompareAndSwap(o, n, (volatile UInt32 *)a)
+
+#define	atomic_set_32(a, n) do {					\
+	while (!atomic_test_set_32(a, *a, n))				\
+		;							\
+} while (0)
+
+#define	atomic_test_set_64(a, o, n)					\
+	OSCompareAndSwap64(o, n, (volatile UInt64 *)a)
+
 #define	atomic_set_64(a, n) do {					\
-	while (!OSCompareAndSwap64(*a, n, (volatile UInt64 *)a))	\
+	while (!atomic_test_set_64(a, *a, n))				\
 		;							\
 } while (0)
 
@@ -100,6 +112,14 @@ extern "C" {
 } while (0)
 #endif /* __LP64__ */
 
+#define	atomic_test_set_ptr(a, o, n)					\
+	OSCompareAndSwapPtr(o, n, (void * volatile *)a)
+
+#define	atomic_set_ptr(a, n) do {					\
+	while (!atomic_test_set_ptr(a, *a, n))				\
+		;							\
+} while (0)
+
 #define	atomic_or_8_ov(a, n)						\
 	((u_int8_t) OSBitOrAtomic8(n, (volatile UInt8 *)a))
 
@@ -154,11 +174,13 @@ extern "C" {
 #define	atomic_bitclear_32(a, n)					\
 	atomic_and_32(a, ~(n))
 
+#define	membar_sync	OSMemoryBarrier
+
 /*
  * Use CPU_CACHE_LINE_SIZE instead of MAX_CPU_CACHE_LINE_SIZE, unless
  * wasting space is of no concern.
  */
-#define	MAX_CPU_CACHE_LINE_SIZE	64
+#define	MAX_CPU_CACHE_LINE_SIZE	128
 #define	CPU_CACHE_LINE_SIZE	mcache_cache_line_size()
 
 #ifndef IS_P2ALIGNED
@@ -244,7 +266,7 @@ typedef struct mcache_cpu {
 	int		cc_objs;	/* number of objects in filled bkt */
 	int		cc_pobjs;	/* number of objects in previous bkt */
 	int		cc_bktsize;	/* number of elements in a full bkt */
-} __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE), packed)) mcache_cpu_t;
+} __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE))) mcache_cpu_t;
 
 typedef unsigned int (*mcache_allocfn_t)(void *, mcache_obj_t ***,
     unsigned int, int);
@@ -303,7 +325,8 @@ typedef struct mcache {
 	/*
 	 * Per-CPU layer, aligned at cache line boundary
 	 */
-	mcache_cpu_t	mc_cpu[1];
+	mcache_cpu_t	mc_cpu[1]
+	    __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE)));
 } mcache_t;
 
 #define	MCACHE_ALIGN	8	/* default guaranteed alignment */
@@ -324,22 +347,23 @@ typedef struct mcache {
 
 #define	MCACHE_STACK_DEPTH 16
 
+#define	MCA_TRN_MAX	2		/* Number of transactions to record */
+
 typedef struct mcache_audit {
 	struct mcache_audit *mca_next;	/* next audit struct */
 	void		*mca_addr;	/* address of buffer */
 	mcache_t	*mca_cache;	/* parent cache of the buffer */
-	struct thread	*mca_thread;	/* thread doing transaction */
-	struct thread	*mca_pthread;	/* previous transaction thread */
 	size_t		mca_contents_size; /* size of saved contents */
 	void		*mca_contents;	/* user-specific saved contents */
-	uint32_t	mca_tstamp;	/* transaction timestamp (ms) */
-	uint32_t	mca_ptstamp;	/* prev transaction timestamp (ms) */
-	uint16_t	mca_depth;	/* pc stack depth */
-	uint16_t	mca_pdepth;	/* previous transaction pc stack */
-	void		*mca_stack[MCACHE_STACK_DEPTH];
-	void		*mca_pstack[MCACHE_STACK_DEPTH];
 	void		*mca_uptr;	/* user-specific pointer */
 	uint32_t	mca_uflags;	/* user-specific flags */
+	uint32_t	mca_next_trn;
+	struct mca_trn {
+		struct thread	*mca_thread;	/* thread doing transaction */
+		uint32_t	mca_tstamp;
+		uint16_t	mca_depth;
+		void		*mca_stack[MCACHE_STACK_DEPTH];
+	} mca_trns[MCA_TRN_MAX];
 } mcache_audit_t;
 
 __private_extern__ int assfail(const char *, const char *, int);
@@ -358,7 +382,7 @@ __private_extern__ unsigned int mcache_alloc_ext(mcache_t *, mcache_obj_t **,
     unsigned int, int);
 __private_extern__ void mcache_free_ext(mcache_t *, mcache_obj_t *);
 __private_extern__ void mcache_reap(void);
-__private_extern__ boolean_t mcache_purge_cache(mcache_t *);
+__private_extern__ boolean_t mcache_purge_cache(mcache_t *, boolean_t);
 __private_extern__ void mcache_waiter_inc(mcache_t *);
 __private_extern__ void mcache_waiter_dec(mcache_t *);
 __private_extern__ boolean_t mcache_bkt_isempty(mcache_t *);
@@ -377,6 +401,10 @@ __private_extern__ char *mcache_dump_mca(mcache_audit_t *);
 __private_extern__ void mcache_audit_panic(mcache_audit_t *, void *, size_t,
     int64_t, int64_t);
 
+extern int32_t total_sbmb_cnt;
+extern int32_t total_sbmb_cnt_floor;
+extern int32_t total_sbmb_cnt_peak;
+extern int64_t sbmb_limreached;
 extern mcache_t *mcache_audit_cache;
 
 #ifdef  __cplusplus