xnu-2782.40.9.tar.gz

[apple/xnu.git] / bsd / kern / mcache.c

diff --git a/bsd/kern/mcache.c b/bsd/kern/mcache.c
index 9d5fb63d63cfbe0c81629910fd2880e92dd5e20e..823fbf95e1a5b72c8057c4f6908fc20329311867 100644 (file)
--- a/bsd/kern/mcache.c
+++ b/bsd/kern/mcache.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+ * Copyright (c) 2006-2014 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *


@@ -54,6 +54,7 @@
 #include <kern/zalloc.h>
 #include <kern/cpu_number.h>
 #include <kern/locks.h>
 #include <kern/zalloc.h>
 #include <kern/cpu_number.h>
 #include <kern/locks.h>

+#include <kern/thread_call.h>

 
 #include <libkern/libkern.h>
 #include <libkern/OSAtomic.h>
 
 #include <libkern/libkern.h>
 #include <libkern/OSAtomic.h>


@@ -72,10 +73,10 @@
 
 /* Allocate extra in case we need to manually align the pointer */
 #define        MCACHE_ALLOC_SIZE \
 
 /* Allocate extra in case we need to manually align the pointer */
 #define        MCACHE_ALLOC_SIZE \

-       (sizeof (void *) + MCACHE_SIZE(ncpu) + CPU_CACHE_SIZE)
+       (sizeof (void *) + MCACHE_SIZE(ncpu) + CPU_CACHE_LINE_SIZE)

 
 #define        MCACHE_CPU(c) \
 
 #define        MCACHE_CPU(c) \

-       (mcache_cpu_t *)((char *)(c) + MCACHE_SIZE(cpu_number()))
+       (mcache_cpu_t *)((void *)((char *)(c) + MCACHE_SIZE(cpu_number())))

 
 /*
  * MCACHE_LIST_LOCK() and MCACHE_LIST_UNLOCK() are macros used
 
 /*
  * MCACHE_LIST_LOCK() and MCACHE_LIST_UNLOCK() are macros used


@@ -98,17 +99,16 @@
 #define        MCACHE_UNLOCK(l)        lck_mtx_unlock(l)
 #define        MCACHE_LOCK_TRY(l)      lck_mtx_try_lock(l)
 
 #define        MCACHE_UNLOCK(l)        lck_mtx_unlock(l)
 #define        MCACHE_LOCK_TRY(l)      lck_mtx_try_lock(l)
 

-/* This should be in a header file */
-#define        atomic_add_32(a, n)     ((void) OSAddAtomic(n, (volatile SInt32 *)a))
-

 static int ncpu;
 static int ncpu;

+static unsigned int cache_line_size;

 static lck_mtx_t *mcache_llock;
 static struct thread *mcache_llock_owner;
 static lck_attr_t *mcache_llock_attr;
 static lck_grp_t *mcache_llock_grp;
 static lck_grp_attr_t *mcache_llock_grp_attr;
 static struct zone *mcache_zone;
 static lck_mtx_t *mcache_llock;
 static struct thread *mcache_llock_owner;
 static lck_attr_t *mcache_llock_attr;
 static lck_grp_t *mcache_llock_grp;
 static lck_grp_attr_t *mcache_llock_grp_attr;
 static struct zone *mcache_zone;

-static unsigned int mcache_reap_interval;
+static const uint32_t mcache_reap_interval = 15;
+static const uint32_t mcache_reap_interval_leeway = 2;

 static UInt32 mcache_reaping;
 static int mcache_ready;
 static int mcache_updating;
 static UInt32 mcache_reaping;
 static int mcache_ready;
 static int mcache_updating;


@@ -120,6 +120,8 @@ static unsigned int mcache_flags = MCF_DEBUG;
 static unsigned int mcache_flags = 0;
 #endif
 
 static unsigned int mcache_flags = 0;
 #endif
 

+int mca_trn_max = MCA_TRN_MAX;
+

 #define        DUMP_MCA_BUF_SIZE       512
 static char *mca_dump_buf;
 
 #define        DUMP_MCA_BUF_SIZE       512
 static char *mca_dump_buf;
 


@@ -137,8 +139,8 @@ static mcache_bkttype_t mcache_bkttype[] = {
 };
 
 static mcache_t *mcache_create_common(const char *, size_t, size_t,
 };
 
 static mcache_t *mcache_create_common(const char *, size_t, size_t,

-    mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_notifyfn_t,
-    void *, u_int32_t, int, int);
+    mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_logfn_t,
+    mcache_notifyfn_t, void *, u_int32_t, int, int);

 static unsigned int mcache_slab_alloc(void *, mcache_obj_t ***,
     unsigned int, int);
 static void mcache_slab_free(void *, mcache_obj_t *, boolean_t);
 static unsigned int mcache_slab_alloc(void *, mcache_obj_t ***,
     unsigned int, int);
 static void mcache_slab_free(void *, mcache_obj_t *, boolean_t);


@@ -158,18 +160,21 @@ static void mcache_cache_reap(mcache_t *);
 static void mcache_cache_update(mcache_t *);
 static void mcache_cache_bkt_resize(void *);
 static void mcache_cache_enable(void *);
 static void mcache_cache_update(mcache_t *);
 static void mcache_cache_bkt_resize(void *);
 static void mcache_cache_enable(void *);

-static void mcache_update(void *);
+static void mcache_update(thread_call_param_t __unused, thread_call_param_t __unused);

 static void mcache_update_timeout(void *);
 static void mcache_applyall(void (*)(mcache_t *));
 static void mcache_reap_start(void *);
 static void mcache_reap_done(void *);
 static void mcache_update_timeout(void *);
 static void mcache_applyall(void (*)(mcache_t *));
 static void mcache_reap_start(void *);
 static void mcache_reap_done(void *);

-static void mcache_reap_timeout(void *);
+static void mcache_reap_timeout(thread_call_param_t __unused, thread_call_param_t);

 static void mcache_notify(mcache_t *, u_int32_t);
 static void mcache_purge(void *);
 
 static LIST_HEAD(, mcache) mcache_head;
 mcache_t *mcache_audit_cache;
 
 static void mcache_notify(mcache_t *, u_int32_t);
 static void mcache_purge(void *);
 
 static LIST_HEAD(, mcache) mcache_head;
 mcache_t *mcache_audit_cache;
 

+static thread_call_t mcache_reap_tcall;
+static thread_call_t mcache_update_tcall;
+

 /*
  * Initialize the framework; this is currently called as part of BSD init.
  */
 /*
  * Initialize the framework; this is currently called as part of BSD init.
  */


@@ -180,7 +185,10 @@ mcache_init(void)
        unsigned int i;
        char name[32];
 
        unsigned int i;
        char name[32];
 

+       VERIFY(mca_trn_max >= 2);
+

        ncpu = ml_get_max_cpus();
        ncpu = ml_get_max_cpus();

+       (void) mcache_cache_line_size();        /* prime it */

 
        mcache_llock_grp_attr = lck_grp_attr_alloc_init();
        mcache_llock_grp = lck_grp_alloc_init("mcache.list",
 
        mcache_llock_grp_attr = lck_grp_attr_alloc_init();
        mcache_llock_grp = lck_grp_alloc_init("mcache.list",


@@ -188,10 +196,16 @@ mcache_init(void)
        mcache_llock_attr = lck_attr_alloc_init();
        mcache_llock = lck_mtx_alloc_init(mcache_llock_grp, mcache_llock_attr);
 
        mcache_llock_attr = lck_attr_alloc_init();
        mcache_llock = lck_mtx_alloc_init(mcache_llock_grp, mcache_llock_attr);
 

+       mcache_reap_tcall = thread_call_allocate(mcache_reap_timeout, NULL);
+       mcache_update_tcall = thread_call_allocate(mcache_update, NULL);
+       if (mcache_reap_tcall == NULL || mcache_update_tcall == NULL)
+               panic("mcache_init: thread_call_allocate failed");
+

        mcache_zone = zinit(MCACHE_ALLOC_SIZE, 256 * MCACHE_ALLOC_SIZE,
            PAGE_SIZE, "mcache");
        if (mcache_zone == NULL)
                panic("mcache_init: failed to allocate mcache zone\n");
        mcache_zone = zinit(MCACHE_ALLOC_SIZE, 256 * MCACHE_ALLOC_SIZE,
            PAGE_SIZE, "mcache");
        if (mcache_zone == NULL)
                panic("mcache_init: failed to allocate mcache zone\n");

+       zone_change(mcache_zone, Z_CALLERACCT, FALSE);

 
        LIST_INIT(&mcache_head);
 
 
        LIST_INIT(&mcache_head);
 


@@ -203,15 +217,17 @@ mcache_init(void)
                    (btp->bt_bktsize + 1) * sizeof (void *), 0, 0, MCR_SLEEP);
        }
 
                    (btp->bt_bktsize + 1) * sizeof (void *), 0, 0, MCR_SLEEP);
        }
 

-       PE_parse_boot_arg("mcache_flags", &mcache_flags);
+       PE_parse_boot_argn("mcache_flags", &mcache_flags, sizeof(mcache_flags));

        mcache_flags &= MCF_FLAGS_MASK;
 
        mcache_audit_cache = mcache_create("audit", sizeof (mcache_audit_t),
            0, 0, MCR_SLEEP);
 
        mcache_flags &= MCF_FLAGS_MASK;
 
        mcache_audit_cache = mcache_create("audit", sizeof (mcache_audit_t),
            0, 0, MCR_SLEEP);
 

-       mcache_reap_interval = 15 * hz;

        mcache_applyall(mcache_cache_bkt_enable);
        mcache_ready = 1;
        mcache_applyall(mcache_cache_bkt_enable);
        mcache_ready = 1;

+
+       printf("mcache: %d CPU(s), %d bytes CPU cache line size\n",
+           ncpu, CPU_CACHE_LINE_SIZE);

 }
 
 /*
 }
 
 /*


@@ -223,6 +239,20 @@ mcache_getflags(void)
        return (mcache_flags);
 }
 
        return (mcache_flags);
 }
 

+/*
+ * Return the CPU cache line size.
+ */
+__private_extern__ unsigned int
+mcache_cache_line_size(void)
+{
+       if (cache_line_size == 0) {
+               ml_cpu_info_t cpu_info;
+               ml_cpu_get_info(&cpu_info);
+               cache_line_size = cpu_info.cache_line_size;
+       }
+       return (cache_line_size);
+}
+

 /*
  * Create a cache using the zone allocator as the backend slab allocator.
  * The caller may specify any alignment for the object; if it specifies 0
 /*
  * Create a cache using the zone allocator as the backend slab allocator.
  * The caller may specify any alignment for the object; if it specifies 0


@@ -233,7 +263,8 @@ mcache_create(const char *name, size_t bufsize, size_t align,
     u_int32_t flags, int wait)
 {
        return (mcache_create_common(name, bufsize, align, mcache_slab_alloc,
     u_int32_t flags, int wait)
 {
        return (mcache_create_common(name, bufsize, align, mcache_slab_alloc,

-           mcache_slab_free, mcache_slab_audit, NULL, NULL, flags, 1, wait));
+           mcache_slab_free, mcache_slab_audit, NULL, NULL, NULL, flags, 1,
+           wait));

 }
 
 /*
 }
 
 /*


@@ -244,10 +275,11 @@ mcache_create(const char *name, size_t bufsize, size_t align,
 __private_extern__ mcache_t *
 mcache_create_ext(const char *name, size_t bufsize,
     mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,
 __private_extern__ mcache_t *
 mcache_create_ext(const char *name, size_t bufsize,
     mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,

-    mcache_notifyfn_t notifyfn, void *arg, u_int32_t flags, int wait)
+    mcache_logfn_t logfn, mcache_notifyfn_t notifyfn, void *arg,
+    u_int32_t flags, int wait)

 {
        return (mcache_create_common(name, bufsize, 0, allocfn,
 {
        return (mcache_create_common(name, bufsize, 0, allocfn,

-           freefn, auditfn, notifyfn, arg, flags, 0, wait));
+           freefn, auditfn, logfn, notifyfn, arg, flags, 0, wait));

 }
 
 /*
 }
 
 /*


@@ -256,8 +288,8 @@ mcache_create_ext(const char *name, size_t bufsize,
 static mcache_t *
 mcache_create_common(const char *name, size_t bufsize, size_t align,
     mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,
 static mcache_t *
 mcache_create_common(const char *name, size_t bufsize, size_t align,
     mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,

-    mcache_notifyfn_t notifyfn, void *arg, u_int32_t flags, int need_zone,
-    int wait)
+    mcache_logfn_t logfn, mcache_notifyfn_t notifyfn, void *arg,
+    u_int32_t flags, int need_zone, int wait)

 {
        mcache_bkttype_t *btp;
        mcache_t *cp = NULL;
 {
        mcache_bkttype_t *btp;
        mcache_t *cp = NULL;


@@ -267,7 +299,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
        char lck_name[64];
 
        /* If auditing is on and print buffer is NULL, allocate it now */
        char lck_name[64];
 
        /* If auditing is on and print buffer is NULL, allocate it now */

-       if ((flags & MCF_AUDIT) && mca_dump_buf == NULL) {
+       if ((flags & MCF_DEBUG) && mca_dump_buf == NULL) {

                int malloc_wait = (wait & MCR_NOSLEEP) ? M_NOWAIT : M_WAITOK;
                MALLOC(mca_dump_buf, char *, DUMP_MCA_BUF_SIZE, M_TEMP,
                    malloc_wait | M_ZERO);
                int malloc_wait = (wait & MCR_NOSLEEP) ? M_NOWAIT : M_WAITOK;
                MALLOC(mca_dump_buf, char *, DUMP_MCA_BUF_SIZE, M_TEMP,
                    malloc_wait | M_ZERO);


@@ -293,7 +325,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
         * is okay since we've allocated extra space for this.
         */
        cp = (mcache_t *)
         * is okay since we've allocated extra space for this.
         */
        cp = (mcache_t *)

-           P2ROUNDUP((intptr_t)buf + sizeof (void *), CPU_CACHE_SIZE);
+           P2ROUNDUP((intptr_t)buf + sizeof (void *), CPU_CACHE_LINE_SIZE);

        pbuf = (void **)((intptr_t)cp - sizeof (void *));
        *pbuf = buf;
 
        pbuf = (void **)((intptr_t)cp - sizeof (void *));
        *pbuf = buf;
 


@@ -313,6 +345,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
        cp->mc_slab_alloc = allocfn;
        cp->mc_slab_free = freefn;
        cp->mc_slab_audit = auditfn;
        cp->mc_slab_alloc = allocfn;
        cp->mc_slab_free = freefn;
        cp->mc_slab_audit = auditfn;

+       cp->mc_slab_log = logfn;

        cp->mc_slab_notify = notifyfn;
        cp->mc_private = need_zone ? cp : arg;
        cp->mc_bufsize = bufsize;
        cp->mc_slab_notify = notifyfn;
        cp->mc_private = need_zone ? cp : arg;
        cp->mc_bufsize = bufsize;


@@ -377,7 +410,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
        for (c = 0; c < ncpu; c++) {
                mcache_cpu_t *ccp = &cp->mc_cpu[c];
 
        for (c = 0; c < ncpu; c++) {
                mcache_cpu_t *ccp = &cp->mc_cpu[c];
 

-               VERIFY(IS_P2ALIGNED(ccp, CPU_CACHE_SIZE));
+               VERIFY(IS_P2ALIGNED(ccp, CPU_CACHE_LINE_SIZE));

                lck_mtx_init(&ccp->cc_lock, cp->mc_cpu_lock_grp,
                    cp->mc_cpu_lock_attr);
                ccp->cc_objs = -1;
                lck_mtx_init(&ccp->cc_lock, cp->mc_cpu_lock_grp,
                    cp->mc_cpu_lock_attr);
                ccp->cc_objs = -1;


@@ -467,6 +500,11 @@ retry_alloc:
                        /* If we got them all, return to caller */
                        if ((need -= objs) == 0) {
                                MCACHE_UNLOCK(&ccp->cc_lock);
                        /* If we got them all, return to caller */
                        if ((need -= objs) == 0) {
                                MCACHE_UNLOCK(&ccp->cc_lock);

+
+                               if (!(cp->mc_flags & MCF_NOLEAKLOG) &&
+                                   cp->mc_slab_log != NULL)
+                                       (*cp->mc_slab_log)(num, *top, TRUE);
+

                                if (cp->mc_flags & MCF_DEBUG)
                                        goto debug_alloc;
 
                                if (cp->mc_flags & MCF_DEBUG)
                                        goto debug_alloc;
 


@@ -534,11 +572,14 @@ retry_alloc:
                }
        }
 
                }
        }
 

+       if (!(cp->mc_flags & MCF_NOLEAKLOG) && cp->mc_slab_log != NULL)
+               (*cp->mc_slab_log)((num - need), *top, TRUE);
+

        if (!(cp->mc_flags & MCF_DEBUG))
                return (num - need);
 
 debug_alloc:
        if (!(cp->mc_flags & MCF_DEBUG))
                return (num - need);
 
 debug_alloc:

-       if (cp->mc_flags & MCF_VERIFY) {
+       if (cp->mc_flags & MCF_DEBUG) {

                mcache_obj_t **o = top;
                unsigned int n;
 
                mcache_obj_t **o = top;
                unsigned int n;
 


@@ -561,7 +602,7 @@ debug_alloc:
        }
 
        /* Invoke the slab layer audit callback if auditing is enabled */
        }
 
        /* Invoke the slab layer audit callback if auditing is enabled */

-       if ((cp->mc_flags & MCF_AUDIT) && cp->mc_slab_audit != NULL)
+       if ((cp->mc_flags & MCF_DEBUG) && cp->mc_slab_audit != NULL)

                (*cp->mc_slab_audit)(cp->mc_private, *top, TRUE);
 
        return (num - need);
                (*cp->mc_slab_audit)(cp->mc_private, *top, TRUE);
 
        return (num - need);


@@ -631,11 +672,10 @@ mcache_purge(void *arg)
        lck_mtx_lock_spin(&cp->mc_sync_lock);
        cp->mc_enable_cnt++;
        lck_mtx_unlock(&cp->mc_sync_lock);
        lck_mtx_lock_spin(&cp->mc_sync_lock);
        cp->mc_enable_cnt++;
        lck_mtx_unlock(&cp->mc_sync_lock);

-

 }
 
 __private_extern__ boolean_t
 }
 
 __private_extern__ boolean_t

-mcache_purge_cache(mcache_t *cp)
+mcache_purge_cache(mcache_t *cp, boolean_t async)

 {
        /*
         * Purging a cache that has no per-CPU caches or is already
 {
        /*
         * Purging a cache that has no per-CPU caches or is already


@@ -652,7 +692,10 @@ mcache_purge_cache(mcache_t *cp)
        cp->mc_purge_cnt++;
        lck_mtx_unlock(&cp->mc_sync_lock);
 
        cp->mc_purge_cnt++;
        lck_mtx_unlock(&cp->mc_sync_lock);
 

-       mcache_dispatch(mcache_purge, cp);
+       if (async)
+               mcache_dispatch(mcache_purge, cp);
+       else
+               mcache_purge(cp);

 
        return (TRUE);
 }
 
        return (TRUE);
 }


@@ -678,8 +721,11 @@ mcache_free_ext(mcache_t *cp, mcache_obj_t *list)
        mcache_obj_t *nlist;
        mcache_bkt_t *bkt;
 
        mcache_obj_t *nlist;
        mcache_bkt_t *bkt;
 

+       if (!(cp->mc_flags & MCF_NOLEAKLOG) && cp->mc_slab_log != NULL)
+               (*cp->mc_slab_log)(0, list, FALSE);
+

        /* Invoke the slab layer audit callback if auditing is enabled */
        /* Invoke the slab layer audit callback if auditing is enabled */

-       if ((cp->mc_flags & MCF_AUDIT) && cp->mc_slab_audit != NULL)
+       if ((cp->mc_flags & MCF_DEBUG) && cp->mc_slab_audit != NULL)

                (*cp->mc_slab_audit)(cp->mc_private, list, FALSE);
 
        MCACHE_LOCK(&ccp->cc_lock);
                (*cp->mc_slab_audit)(cp->mc_private, list, FALSE);
 
        MCACHE_LOCK(&ccp->cc_lock);


@@ -899,7 +945,7 @@ mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait)
                 * the nearest 64-bit multiply; this is because we use
                 * 64-bit memory access to set/check the pattern.
                 */
                 * the nearest 64-bit multiply; this is because we use
                 * 64-bit memory access to set/check the pattern.
                 */

-               if (flags & MCF_AUDIT) {
+               if (flags & MCF_DEBUG) {

                        VERIFY(((intptr_t)base + rsize) <=
                            ((intptr_t)buf + cp->mc_chunksize));
                        mcache_set_pattern(MCACHE_FREE_PATTERN, base, rsize);
                        VERIFY(((intptr_t)base + rsize) <=
                            ((intptr_t)buf + cp->mc_chunksize));
                        mcache_set_pattern(MCACHE_FREE_PATTERN, base, rsize);


@@ -958,7 +1004,7 @@ mcache_slab_free(void *arg, mcache_obj_t *list, __unused boolean_t purged)
                /* Get the original address since we're about to free it */
                pbuf = (void **)((intptr_t)base - sizeof (void *));
 
                /* Get the original address since we're about to free it */
                pbuf = (void **)((intptr_t)base - sizeof (void *));
 

-               if (flags & MCF_AUDIT) {
+               if (flags & MCF_DEBUG) {

                        VERIFY(((intptr_t)base + rsize) <=
                            ((intptr_t)*pbuf + cp->mc_chunksize));
                        mcache_audit_free_verify(NULL, base, offset, rsize);
                        VERIFY(((intptr_t)base + rsize) <=
                            ((intptr_t)*pbuf + cp->mc_chunksize));
                        mcache_audit_free_verify(NULL, base, offset, rsize);


@@ -1156,7 +1202,7 @@ mcache_bkt_destroy(mcache_t *cp, mcache_bkttype_t *btp, mcache_bkt_t *bkt,
        if (nobjs > 0) {
                mcache_obj_t *top = bkt->bkt_obj[nobjs - 1];
 
        if (nobjs > 0) {
                mcache_obj_t *top = bkt->bkt_obj[nobjs - 1];
 

-               if (cp->mc_flags & MCF_VERIFY) {
+               if (cp->mc_flags & MCF_DEBUG) {

                        mcache_obj_t *o = top;
                        int cnt = 0;
 
                        mcache_obj_t *o = top;
                        int cnt = 0;
 


@@ -1222,7 +1268,8 @@ mcache_bkt_ws_reap(mcache_t *cp)
 }
 
 static void
 }
 
 static void

-mcache_reap_timeout(void *arg)
+mcache_reap_timeout(thread_call_param_t dummy __unused,
+    thread_call_param_t arg)

 {
        volatile UInt32 *flag = arg;
 
 {
        volatile UInt32 *flag = arg;
 


@@ -1234,7 +1281,14 @@ mcache_reap_timeout(void *arg)
 static void
 mcache_reap_done(void *flag)
 {
 static void
 mcache_reap_done(void *flag)
 {

-       timeout(mcache_reap_timeout, flag, mcache_reap_interval);
+       uint64_t deadline, leeway;
+
+       clock_interval_to_deadline(mcache_reap_interval, NSEC_PER_SEC,
+           &deadline);
+       clock_interval_to_absolutetime_interval(mcache_reap_interval_leeway,
+           NSEC_PER_SEC, &leeway);
+       thread_call_enter_delayed_with_leeway(mcache_reap_tcall, flag,
+           deadline, leeway, THREAD_CALL_DELAY_LEEWAY);

 }
 
 static void
 }
 
 static void


@@ -1359,14 +1413,22 @@ mcache_cache_enable(void *arg)
 static void
 mcache_update_timeout(__unused void *arg)
 {
 static void
 mcache_update_timeout(__unused void *arg)
 {

-       timeout(mcache_update, NULL, mcache_reap_interval);
+       uint64_t deadline, leeway;
+
+       clock_interval_to_deadline(mcache_reap_interval, NSEC_PER_SEC,
+           &deadline);
+       clock_interval_to_absolutetime_interval(mcache_reap_interval_leeway,
+           NSEC_PER_SEC, &leeway);
+       thread_call_enter_delayed_with_leeway(mcache_update_tcall, NULL,
+           deadline, leeway, THREAD_CALL_DELAY_LEEWAY);

 }
 
 static void
 }
 
 static void

-mcache_update(__unused void *arg)
+mcache_update(thread_call_param_t arg __unused,
+    thread_call_param_t dummy __unused)

 {
        mcache_applyall(mcache_cache_update);
 {
        mcache_applyall(mcache_cache_update);

-       mcache_dispatch(mcache_update_timeout, NULL);
+       mcache_update_timeout(NULL);

 }
 
 static void
 }
 
 static void


@@ -1389,22 +1451,41 @@ mcache_dispatch(void (*func)(void *), void *arg)
 }
 
 __private_extern__ void
 }
 
 __private_extern__ void

-mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp)
+mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp,
+    struct timeval *base_ts)

 {
 {

+       struct timeval now, base = { 0, 0 };
+       void *stack[MCACHE_STACK_DEPTH + 1];
+       struct mca_trn *transaction;
+
+       transaction = &mca->mca_trns[mca->mca_next_trn];
+

        mca->mca_addr = addr;
        mca->mca_cache = cp;
        mca->mca_addr = addr;
        mca->mca_cache = cp;

-       mca->mca_pthread = mca->mca_thread;
-       mca->mca_thread = current_thread();
-       bcopy(mca->mca_stack, mca->mca_pstack, sizeof (mca->mca_pstack));
-       mca->mca_pdepth = mca->mca_depth;
-       bzero(mca->mca_stack, sizeof (mca->mca_stack));
-       mca->mca_depth = OSBacktrace(mca->mca_stack, MCACHE_STACK_DEPTH);
+
+       transaction->mca_thread = current_thread();
+
+       bzero(stack, sizeof (stack));
+       transaction->mca_depth = OSBacktrace(stack, MCACHE_STACK_DEPTH + 1) - 1;
+       bcopy(&stack[1], transaction->mca_stack,
+               sizeof (transaction->mca_stack));
+
+       microuptime(&now);
+       if (base_ts != NULL)
+               base = *base_ts;
+       /* tstamp is in ms relative to base_ts */
+       transaction->mca_tstamp = ((now.tv_usec - base.tv_usec) / 1000);
+       if ((now.tv_sec - base.tv_sec) > 0)
+               transaction->mca_tstamp += ((now.tv_sec - base.tv_sec) * 1000);
+
+       mca->mca_next_trn =
+               (mca->mca_next_trn + 1) % mca_trn_max;

 }
 
 __private_extern__ void
 mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 {
 }
 
 __private_extern__ void
 mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 {

-       u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size);
+       u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));

        u_int64_t *buf = (u_int64_t *)buf_arg;
 
        VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));
        u_int64_t *buf = (u_int64_t *)buf_arg;
 
        VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));


@@ -1417,7 +1498,7 @@ mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 __private_extern__ void *
 mcache_verify_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 {
 __private_extern__ void *
 mcache_verify_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 {

-       u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size);
+       u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));

        u_int64_t *buf;
 
        VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));
        u_int64_t *buf;
 
        VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));


@@ -1434,7 +1515,7 @@ __private_extern__ void *
 mcache_verify_set_pattern(u_int64_t old, u_int64_t new, void *buf_arg,
     size_t size)
 {
 mcache_verify_set_pattern(u_int64_t old, u_int64_t new, void *buf_arg,
     size_t size)
 {

-       u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size);
+       u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));

        u_int64_t *buf;
 
        VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));
        u_int64_t *buf;
 
        VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));


@@ -1499,7 +1580,27 @@ mcache_audit_free_verify_set(mcache_audit_t *mca, void *base, size_t offset,
        ((mcache_obj_t *)addr)->obj_next = next;
 }
 
        ((mcache_obj_t *)addr)->obj_next = next;
 }
 

-#undef panic(...)
+#undef panic
+
+#define        DUMP_TRN_FMT() \
+           "%s transaction thread %p saved PC stack (%d deep):\n" \
+           "\t%p, %p, %p, %p, %p, %p, %p, %p\n" \
+           "\t%p, %p, %p, %p, %p, %p, %p, %p\n"
+
+#define        DUMP_TRN_FIELDS(s, x) \
+           s, \
+           mca->mca_trns[x].mca_thread, mca->mca_trns[x].mca_depth, \
+           mca->mca_trns[x].mca_stack[0], mca->mca_trns[x].mca_stack[1], \
+           mca->mca_trns[x].mca_stack[2], mca->mca_trns[x].mca_stack[3], \
+           mca->mca_trns[x].mca_stack[4], mca->mca_trns[x].mca_stack[5], \
+           mca->mca_trns[x].mca_stack[6], mca->mca_trns[x].mca_stack[7], \
+           mca->mca_trns[x].mca_stack[8], mca->mca_trns[x].mca_stack[9], \
+           mca->mca_trns[x].mca_stack[10], mca->mca_trns[x].mca_stack[11], \
+           mca->mca_trns[x].mca_stack[12], mca->mca_trns[x].mca_stack[13], \
+           mca->mca_trns[x].mca_stack[14], mca->mca_trns[x].mca_stack[15]
+
+#define        MCA_TRN_LAST ((mca->mca_next_trn + mca_trn_max) % mca_trn_max)
+#define        MCA_TRN_PREV ((mca->mca_next_trn + mca_trn_max - 1) % mca_trn_max)

 
 __private_extern__ char *
 mcache_dump_mca(mcache_audit_t *mca)
 
 __private_extern__ char *
 mcache_dump_mca(mcache_audit_t *mca)


@@ -1508,29 +1609,16 @@ mcache_dump_mca(mcache_audit_t *mca)
                return (NULL);
 
        snprintf(mca_dump_buf, DUMP_MCA_BUF_SIZE,
                return (NULL);
 
        snprintf(mca_dump_buf, DUMP_MCA_BUF_SIZE,

-           "mca %p: addr %p, cache %p (%s)\n"
-           "last transaction; thread %p, saved PC stack (%d deep):\n"
-           "\t%p, %p, %p, %p, %p, %p, %p, %p\n"
-           "\t%p, %p, %p, %p, %p, %p, %p, %p\n"
-           "previous transaction; thread %p, saved PC stack (%d deep):\n"
-           "\t%p, %p, %p, %p, %p, %p, %p, %p\n"
-           "\t%p, %p, %p, %p, %p, %p, %p, %p\n",
+           "mca %p: addr %p, cache %p (%s) nxttrn %d\n"
+           DUMP_TRN_FMT()
+           DUMP_TRN_FMT(),
+

            mca, mca->mca_addr, mca->mca_cache,
            mca->mca_cache ? mca->mca_cache->mc_name : "?",
            mca, mca->mca_addr, mca->mca_cache,
            mca->mca_cache ? mca->mca_cache->mc_name : "?",

-           mca->mca_thread, mca->mca_depth,
-           mca->mca_stack[0], mca->mca_stack[1], mca->mca_stack[2],
-           mca->mca_stack[3], mca->mca_stack[4], mca->mca_stack[5],
-           mca->mca_stack[6], mca->mca_stack[7], mca->mca_stack[8],
-           mca->mca_stack[9], mca->mca_stack[10], mca->mca_stack[11],
-           mca->mca_stack[12], mca->mca_stack[13], mca->mca_stack[14],
-           mca->mca_stack[15],
-           mca->mca_pthread, mca->mca_pdepth,
-           mca->mca_pstack[0], mca->mca_pstack[1], mca->mca_pstack[2],
-           mca->mca_pstack[3], mca->mca_pstack[4], mca->mca_pstack[5],
-           mca->mca_pstack[6], mca->mca_pstack[7], mca->mca_pstack[8],
-           mca->mca_pstack[9], mca->mca_pstack[10], mca->mca_pstack[11],
-           mca->mca_pstack[12], mca->mca_pstack[13], mca->mca_pstack[14],
-           mca->mca_pstack[15]);
+           mca->mca_next_trn,
+
+           DUMP_TRN_FIELDS("last", MCA_TRN_LAST),
+           DUMP_TRN_FIELDS("previous", MCA_TRN_PREV));

 
        return (mca_dump_buf);
 }
 
        return (mca_dump_buf);
 }