X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/b0d623f7f2ae71ed96e60569f61f9a9a27016e80..15129b1c8dbb3650c63b70adb1cad9af601c6c17:/bsd/kern/mcache.c?ds=inline

diff --git a/bsd/kern/mcache.c b/bsd/kern/mcache.c
index 14416f34a..0ffea0719 100644
--- a/bsd/kern/mcache.c
+++ b/bsd/kern/mcache.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+ * Copyright (c) 2006-2013 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
@@ -72,10 +72,10 @@
 
 /* Allocate extra in case we need to manually align the pointer */
 #define	MCACHE_ALLOC_SIZE \
-	(sizeof (void *) + MCACHE_SIZE(ncpu) + CPU_CACHE_SIZE)
+	(sizeof (void *) + MCACHE_SIZE(ncpu) + CPU_CACHE_LINE_SIZE)
 
 #define	MCACHE_CPU(c) \
-	(mcache_cpu_t *)((char *)(c) + MCACHE_SIZE(cpu_number()))
+	(mcache_cpu_t *)((void *)((char *)(c) + MCACHE_SIZE(cpu_number())))
 
 /*
  * MCACHE_LIST_LOCK() and MCACHE_LIST_UNLOCK() are macros used
@@ -98,10 +98,8 @@
 #define	MCACHE_UNLOCK(l)	lck_mtx_unlock(l)
 #define	MCACHE_LOCK_TRY(l)	lck_mtx_try_lock(l)
 
-/* This should be in a header file */
-#define	atomic_add_32(a, n)	((void) OSAddAtomic(n, a))
-
 static int ncpu;
+static unsigned int cache_line_size;
 static lck_mtx_t *mcache_llock;
 static struct thread *mcache_llock_owner;
 static lck_attr_t *mcache_llock_attr;
@@ -137,8 +135,8 @@ static mcache_bkttype_t mcache_bkttype[] = {
 };
 
 static mcache_t *mcache_create_common(const char *, size_t, size_t,
-    mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_notifyfn_t,
-    void *, u_int32_t, int, int);
+    mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_logfn_t,
+    mcache_notifyfn_t, void *, u_int32_t, int, int);
 static unsigned int mcache_slab_alloc(void *, mcache_obj_t ***,
     unsigned int, int);
 static void mcache_slab_free(void *, mcache_obj_t *, boolean_t);
@@ -181,6 +179,7 @@ mcache_init(void)
 	char name[32];
 
 	ncpu = ml_get_max_cpus();
+	(void) mcache_cache_line_size();	/* prime it */
 
 	mcache_llock_grp_attr = lck_grp_attr_alloc_init();
 	mcache_llock_grp = lck_grp_alloc_init("mcache.list",
@@ -192,6 +191,7 @@ mcache_init(void)
 	    PAGE_SIZE, "mcache");
 	if (mcache_zone == NULL)
 		panic("mcache_init: failed to allocate mcache zone\n");
+	zone_change(mcache_zone, Z_CALLERACCT, FALSE);
 
 	LIST_INIT(&mcache_head);
 
@@ -212,6 +212,9 @@ mcache_init(void)
 	mcache_reap_interval = 15 * hz;
 	mcache_applyall(mcache_cache_bkt_enable);
 	mcache_ready = 1;
+
+	printf("mcache: %d CPU(s), %d bytes CPU cache line size\n",
+	    ncpu, CPU_CACHE_LINE_SIZE);
 }
 
 /*
@@ -223,6 +226,20 @@ mcache_getflags(void)
 	return (mcache_flags);
 }
 
+/*
+ * Return the CPU cache line size.
+ */
+__private_extern__ unsigned int
+mcache_cache_line_size(void)
+{
+	if (cache_line_size == 0) {
+		ml_cpu_info_t cpu_info;
+		ml_cpu_get_info(&cpu_info);
+		cache_line_size = cpu_info.cache_line_size;
+	}
+	return (cache_line_size);
+}
+
 /*
  * Create a cache using the zone allocator as the backend slab allocator.
  * The caller may specify any alignment for the object; if it specifies 0
@@ -233,7 +250,8 @@ mcache_create(const char *name, size_t bufsize, size_t align,
     u_int32_t flags, int wait)
 {
 	return (mcache_create_common(name, bufsize, align, mcache_slab_alloc,
-	    mcache_slab_free, mcache_slab_audit, NULL, NULL, flags, 1, wait));
+	    mcache_slab_free, mcache_slab_audit, NULL, NULL, NULL, flags, 1,
+	    wait));
 }
 
 /*
@@ -244,10 +262,11 @@ mcache_create(const char *name, size_t bufsize, size_t align,
 __private_extern__ mcache_t *
 mcache_create_ext(const char *name, size_t bufsize,
     mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,
-    mcache_notifyfn_t notifyfn, void *arg, u_int32_t flags, int wait)
+    mcache_logfn_t logfn, mcache_notifyfn_t notifyfn, void *arg,
+    u_int32_t flags, int wait)
 {
 	return (mcache_create_common(name, bufsize, 0, allocfn,
-	    freefn, auditfn, notifyfn, arg, flags, 0, wait));
+	    freefn, auditfn, logfn, notifyfn, arg, flags, 0, wait));
 }
 
 /*
@@ -256,8 +275,8 @@ mcache_create_ext(const char *name, size_t bufsize,
 static mcache_t *
 mcache_create_common(const char *name, size_t bufsize, size_t align,
     mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,
-    mcache_notifyfn_t notifyfn, void *arg, u_int32_t flags, int need_zone,
-    int wait)
+    mcache_logfn_t logfn, mcache_notifyfn_t notifyfn, void *arg,
+    u_int32_t flags, int need_zone, int wait)
 {
 	mcache_bkttype_t *btp;
 	mcache_t *cp = NULL;
@@ -267,7 +286,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
 	char lck_name[64];
 
 	/* If auditing is on and print buffer is NULL, allocate it now */
-	if ((flags & MCF_AUDIT) && mca_dump_buf == NULL) {
+	if ((flags & MCF_DEBUG) && mca_dump_buf == NULL) {
 		int malloc_wait = (wait & MCR_NOSLEEP) ? M_NOWAIT : M_WAITOK;
 		MALLOC(mca_dump_buf, char *, DUMP_MCA_BUF_SIZE, M_TEMP,
 		    malloc_wait | M_ZERO);
@@ -293,7 +312,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
 	 * is okay since we've allocated extra space for this.
 	 */
 	cp = (mcache_t *)
-	    P2ROUNDUP((intptr_t)buf + sizeof (void *), CPU_CACHE_SIZE);
+	    P2ROUNDUP((intptr_t)buf + sizeof (void *), CPU_CACHE_LINE_SIZE);
 	pbuf = (void **)((intptr_t)cp - sizeof (void *));
 	*pbuf = buf;
 
@@ -313,6 +332,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
 	cp->mc_slab_alloc = allocfn;
 	cp->mc_slab_free = freefn;
 	cp->mc_slab_audit = auditfn;
+	cp->mc_slab_log = logfn;
 	cp->mc_slab_notify = notifyfn;
 	cp->mc_private = need_zone ? cp : arg;
 	cp->mc_bufsize = bufsize;
@@ -377,7 +397,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
 	for (c = 0; c < ncpu; c++) {
 		mcache_cpu_t *ccp = &cp->mc_cpu[c];
 
-		VERIFY(IS_P2ALIGNED(ccp, CPU_CACHE_SIZE));
+		VERIFY(IS_P2ALIGNED(ccp, CPU_CACHE_LINE_SIZE));
 		lck_mtx_init(&ccp->cc_lock, cp->mc_cpu_lock_grp,
 		    cp->mc_cpu_lock_attr);
 		ccp->cc_objs = -1;
@@ -467,6 +487,11 @@ retry_alloc:
 			/* If we got them all, return to caller */
 			if ((need -= objs) == 0) {
 				MCACHE_UNLOCK(&ccp->cc_lock);
+
+				if (!(cp->mc_flags & MCF_NOLEAKLOG) &&
+				    cp->mc_slab_log != NULL)
+					(*cp->mc_slab_log)(num, *top, TRUE);
+
 				if (cp->mc_flags & MCF_DEBUG)
 					goto debug_alloc;
 
@@ -534,11 +559,14 @@ retry_alloc:
 		}
 	}
 
+	if (!(cp->mc_flags & MCF_NOLEAKLOG) && cp->mc_slab_log != NULL)
+		(*cp->mc_slab_log)((num - need), *top, TRUE);
+
 	if (!(cp->mc_flags & MCF_DEBUG))
 		return (num - need);
 
 debug_alloc:
-	if (cp->mc_flags & MCF_VERIFY) {
+	if (cp->mc_flags & MCF_DEBUG) {
 		mcache_obj_t **o = top;
 		unsigned int n;
 
@@ -561,7 +589,7 @@ debug_alloc:
 	}
 
 	/* Invoke the slab layer audit callback if auditing is enabled */
-	if ((cp->mc_flags & MCF_AUDIT) && cp->mc_slab_audit != NULL)
+	if ((cp->mc_flags & MCF_DEBUG) && cp->mc_slab_audit != NULL)
 		(*cp->mc_slab_audit)(cp->mc_private, *top, TRUE);
 
 	return (num - need);
@@ -678,8 +706,11 @@ mcache_free_ext(mcache_t *cp, mcache_obj_t *list)
 	mcache_obj_t *nlist;
 	mcache_bkt_t *bkt;
 
+	if (!(cp->mc_flags & MCF_NOLEAKLOG) && cp->mc_slab_log != NULL)
+		(*cp->mc_slab_log)(0, list, FALSE);
+
 	/* Invoke the slab layer audit callback if auditing is enabled */
-	if ((cp->mc_flags & MCF_AUDIT) && cp->mc_slab_audit != NULL)
+	if ((cp->mc_flags & MCF_DEBUG) && cp->mc_slab_audit != NULL)
 		(*cp->mc_slab_audit)(cp->mc_private, list, FALSE);
 
 	MCACHE_LOCK(&ccp->cc_lock);
@@ -899,7 +930,7 @@ mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait)
 		 * the nearest 64-bit multiply; this is because we use
 		 * 64-bit memory access to set/check the pattern.
 		 */
-		if (flags & MCF_AUDIT) {
+		if (flags & MCF_DEBUG) {
 			VERIFY(((intptr_t)base + rsize) <=
 			    ((intptr_t)buf + cp->mc_chunksize));
 			mcache_set_pattern(MCACHE_FREE_PATTERN, base, rsize);
@@ -958,7 +989,7 @@ mcache_slab_free(void *arg, mcache_obj_t *list, __unused boolean_t purged)
 		/* Get the original address since we're about to free it */
 		pbuf = (void **)((intptr_t)base - sizeof (void *));
 
-		if (flags & MCF_AUDIT) {
+		if (flags & MCF_DEBUG) {
 			VERIFY(((intptr_t)base + rsize) <=
 			    ((intptr_t)*pbuf + cp->mc_chunksize));
 			mcache_audit_free_verify(NULL, base, offset, rsize);
@@ -1156,7 +1187,7 @@ mcache_bkt_destroy(mcache_t *cp, mcache_bkttype_t *btp, mcache_bkt_t *bkt,
 	if (nobjs > 0) {
 		mcache_obj_t *top = bkt->bkt_obj[nobjs - 1];
 
-		if (cp->mc_flags & MCF_VERIFY) {
+		if (cp->mc_flags & MCF_DEBUG) {
 			mcache_obj_t *o = top;
 			int cnt = 0;
 
@@ -1389,22 +1420,36 @@ mcache_dispatch(void (*func)(void *), void *arg)
 }
 
 __private_extern__ void
-mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp)
+mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp,
+    struct timeval *base_ts)
 {
+	struct timeval now, base = { 0, 0 };
+	void *stack[MCACHE_STACK_DEPTH + 1];
+
 	mca->mca_addr = addr;
 	mca->mca_cache = cp;
 	mca->mca_pthread = mca->mca_thread;
 	mca->mca_thread = current_thread();
 	bcopy(mca->mca_stack, mca->mca_pstack, sizeof (mca->mca_pstack));
 	mca->mca_pdepth = mca->mca_depth;
-	bzero(mca->mca_stack, sizeof (mca->mca_stack));
-	mca->mca_depth = OSBacktrace(mca->mca_stack, MCACHE_STACK_DEPTH);
+	bzero(stack, sizeof (stack));
+	mca->mca_depth = OSBacktrace(stack, MCACHE_STACK_DEPTH + 1) - 1;
+	bcopy(&stack[1], mca->mca_stack, sizeof (mca->mca_pstack));
+
+	mca->mca_ptstamp = mca->mca_tstamp;
+	microuptime(&now);
+	if (base_ts != NULL)
+		base = *base_ts;
+	/* tstamp is in ms relative to base_ts */
+	mca->mca_tstamp = ((now.tv_usec - base.tv_usec) / 1000);
+	if ((now.tv_sec - base.tv_sec) > 0)
+		mca->mca_tstamp += ((now.tv_sec - base.tv_sec) * 1000);
 }
 
 __private_extern__ void
 mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 {
-	u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size);
+	u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));
 	u_int64_t *buf = (u_int64_t *)buf_arg;
 
 	VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));
@@ -1417,7 +1462,7 @@ mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 __private_extern__ void *
 mcache_verify_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 {
-	u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size);
+	u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));
 	u_int64_t *buf;
 
 	VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));
@@ -1434,7 +1479,7 @@ __private_extern__ void *
 mcache_verify_set_pattern(u_int64_t old, u_int64_t new, void *buf_arg,
     size_t size)
 {
-	u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size);
+	u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));
 	u_int64_t *buf;
 
 	VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));