X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/b0d623f7f2ae71ed96e60569f61f9a9a27016e80..813fb2f63a553c957e917ede5f119b021d6ce391:/bsd/kern/mcache.c diff --git a/bsd/kern/mcache.c b/bsd/kern/mcache.c index 14416f34a..823fbf95e 100644 --- a/bsd/kern/mcache.c +++ b/bsd/kern/mcache.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2007 Apple Inc. All rights reserved. + * Copyright (c) 2006-2014 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -54,6 +54,7 @@ #include #include #include +#include #include #include @@ -72,10 +73,10 @@ /* Allocate extra in case we need to manually align the pointer */ #define MCACHE_ALLOC_SIZE \ - (sizeof (void *) + MCACHE_SIZE(ncpu) + CPU_CACHE_SIZE) + (sizeof (void *) + MCACHE_SIZE(ncpu) + CPU_CACHE_LINE_SIZE) #define MCACHE_CPU(c) \ - (mcache_cpu_t *)((char *)(c) + MCACHE_SIZE(cpu_number())) + (mcache_cpu_t *)((void *)((char *)(c) + MCACHE_SIZE(cpu_number()))) /* * MCACHE_LIST_LOCK() and MCACHE_LIST_UNLOCK() are macros used @@ -98,17 +99,16 @@ #define MCACHE_UNLOCK(l) lck_mtx_unlock(l) #define MCACHE_LOCK_TRY(l) lck_mtx_try_lock(l) -/* This should be in a header file */ -#define atomic_add_32(a, n) ((void) OSAddAtomic(n, a)) - static int ncpu; +static unsigned int cache_line_size; static lck_mtx_t *mcache_llock; static struct thread *mcache_llock_owner; static lck_attr_t *mcache_llock_attr; static lck_grp_t *mcache_llock_grp; static lck_grp_attr_t *mcache_llock_grp_attr; static struct zone *mcache_zone; -static unsigned int mcache_reap_interval; +static const uint32_t mcache_reap_interval = 15; +static const uint32_t mcache_reap_interval_leeway = 2; static UInt32 mcache_reaping; static int mcache_ready; static int mcache_updating; @@ -120,6 +120,8 @@ static unsigned int mcache_flags = MCF_DEBUG; static unsigned int mcache_flags = 0; #endif +int mca_trn_max = MCA_TRN_MAX; + #define DUMP_MCA_BUF_SIZE 512 static char *mca_dump_buf; @@ -137,8 +139,8 @@ static mcache_bkttype_t mcache_bkttype[] = { }; static mcache_t *mcache_create_common(const char *, size_t, size_t, - mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_notifyfn_t, - void *, u_int32_t, int, int); + mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_logfn_t, + mcache_notifyfn_t, void *, u_int32_t, int, int); static unsigned int mcache_slab_alloc(void *, mcache_obj_t ***, unsigned int, int); static void mcache_slab_free(void *, mcache_obj_t *, boolean_t); @@ -158,18 +160,21 @@ static void mcache_cache_reap(mcache_t *); static void mcache_cache_update(mcache_t *); static void mcache_cache_bkt_resize(void *); static void mcache_cache_enable(void *); -static void mcache_update(void *); +static void mcache_update(thread_call_param_t __unused, thread_call_param_t __unused); static void mcache_update_timeout(void *); static void mcache_applyall(void (*)(mcache_t *)); static void mcache_reap_start(void *); static void mcache_reap_done(void *); -static void mcache_reap_timeout(void *); +static void mcache_reap_timeout(thread_call_param_t __unused, thread_call_param_t); static void mcache_notify(mcache_t *, u_int32_t); static void mcache_purge(void *); static LIST_HEAD(, mcache) mcache_head; mcache_t *mcache_audit_cache; +static thread_call_t mcache_reap_tcall; +static thread_call_t mcache_update_tcall; + /* * Initialize the framework; this is currently called as part of BSD init. */ @@ -180,7 +185,10 @@ mcache_init(void) unsigned int i; char name[32]; + VERIFY(mca_trn_max >= 2); + ncpu = ml_get_max_cpus(); + (void) mcache_cache_line_size(); /* prime it */ mcache_llock_grp_attr = lck_grp_attr_alloc_init(); mcache_llock_grp = lck_grp_alloc_init("mcache.list", @@ -188,10 +196,16 @@ mcache_init(void) mcache_llock_attr = lck_attr_alloc_init(); mcache_llock = lck_mtx_alloc_init(mcache_llock_grp, mcache_llock_attr); + mcache_reap_tcall = thread_call_allocate(mcache_reap_timeout, NULL); + mcache_update_tcall = thread_call_allocate(mcache_update, NULL); + if (mcache_reap_tcall == NULL || mcache_update_tcall == NULL) + panic("mcache_init: thread_call_allocate failed"); + mcache_zone = zinit(MCACHE_ALLOC_SIZE, 256 * MCACHE_ALLOC_SIZE, PAGE_SIZE, "mcache"); if (mcache_zone == NULL) panic("mcache_init: failed to allocate mcache zone\n"); + zone_change(mcache_zone, Z_CALLERACCT, FALSE); LIST_INIT(&mcache_head); @@ -203,15 +217,17 @@ mcache_init(void) (btp->bt_bktsize + 1) * sizeof (void *), 0, 0, MCR_SLEEP); } - PE_parse_boot_argn("mcache_flags", &mcache_flags, sizeof (mcache_flags)); + PE_parse_boot_argn("mcache_flags", &mcache_flags, sizeof(mcache_flags)); mcache_flags &= MCF_FLAGS_MASK; mcache_audit_cache = mcache_create("audit", sizeof (mcache_audit_t), 0, 0, MCR_SLEEP); - mcache_reap_interval = 15 * hz; mcache_applyall(mcache_cache_bkt_enable); mcache_ready = 1; + + printf("mcache: %d CPU(s), %d bytes CPU cache line size\n", + ncpu, CPU_CACHE_LINE_SIZE); } /* @@ -223,6 +239,20 @@ mcache_getflags(void) return (mcache_flags); } +/* + * Return the CPU cache line size. + */ +__private_extern__ unsigned int +mcache_cache_line_size(void) +{ + if (cache_line_size == 0) { + ml_cpu_info_t cpu_info; + ml_cpu_get_info(&cpu_info); + cache_line_size = cpu_info.cache_line_size; + } + return (cache_line_size); +} + /* * Create a cache using the zone allocator as the backend slab allocator. * The caller may specify any alignment for the object; if it specifies 0 @@ -233,7 +263,8 @@ mcache_create(const char *name, size_t bufsize, size_t align, u_int32_t flags, int wait) { return (mcache_create_common(name, bufsize, align, mcache_slab_alloc, - mcache_slab_free, mcache_slab_audit, NULL, NULL, flags, 1, wait)); + mcache_slab_free, mcache_slab_audit, NULL, NULL, NULL, flags, 1, + wait)); } /* @@ -244,10 +275,11 @@ mcache_create(const char *name, size_t bufsize, size_t align, __private_extern__ mcache_t * mcache_create_ext(const char *name, size_t bufsize, mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn, - mcache_notifyfn_t notifyfn, void *arg, u_int32_t flags, int wait) + mcache_logfn_t logfn, mcache_notifyfn_t notifyfn, void *arg, + u_int32_t flags, int wait) { return (mcache_create_common(name, bufsize, 0, allocfn, - freefn, auditfn, notifyfn, arg, flags, 0, wait)); + freefn, auditfn, logfn, notifyfn, arg, flags, 0, wait)); } /* @@ -256,8 +288,8 @@ mcache_create_ext(const char *name, size_t bufsize, static mcache_t * mcache_create_common(const char *name, size_t bufsize, size_t align, mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn, - mcache_notifyfn_t notifyfn, void *arg, u_int32_t flags, int need_zone, - int wait) + mcache_logfn_t logfn, mcache_notifyfn_t notifyfn, void *arg, + u_int32_t flags, int need_zone, int wait) { mcache_bkttype_t *btp; mcache_t *cp = NULL; @@ -267,7 +299,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align, char lck_name[64]; /* If auditing is on and print buffer is NULL, allocate it now */ - if ((flags & MCF_AUDIT) && mca_dump_buf == NULL) { + if ((flags & MCF_DEBUG) && mca_dump_buf == NULL) { int malloc_wait = (wait & MCR_NOSLEEP) ? M_NOWAIT : M_WAITOK; MALLOC(mca_dump_buf, char *, DUMP_MCA_BUF_SIZE, M_TEMP, malloc_wait | M_ZERO); @@ -293,7 +325,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align, * is okay since we've allocated extra space for this. */ cp = (mcache_t *) - P2ROUNDUP((intptr_t)buf + sizeof (void *), CPU_CACHE_SIZE); + P2ROUNDUP((intptr_t)buf + sizeof (void *), CPU_CACHE_LINE_SIZE); pbuf = (void **)((intptr_t)cp - sizeof (void *)); *pbuf = buf; @@ -313,6 +345,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align, cp->mc_slab_alloc = allocfn; cp->mc_slab_free = freefn; cp->mc_slab_audit = auditfn; + cp->mc_slab_log = logfn; cp->mc_slab_notify = notifyfn; cp->mc_private = need_zone ? cp : arg; cp->mc_bufsize = bufsize; @@ -377,7 +410,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align, for (c = 0; c < ncpu; c++) { mcache_cpu_t *ccp = &cp->mc_cpu[c]; - VERIFY(IS_P2ALIGNED(ccp, CPU_CACHE_SIZE)); + VERIFY(IS_P2ALIGNED(ccp, CPU_CACHE_LINE_SIZE)); lck_mtx_init(&ccp->cc_lock, cp->mc_cpu_lock_grp, cp->mc_cpu_lock_attr); ccp->cc_objs = -1; @@ -467,6 +500,11 @@ retry_alloc: /* If we got them all, return to caller */ if ((need -= objs) == 0) { MCACHE_UNLOCK(&ccp->cc_lock); + + if (!(cp->mc_flags & MCF_NOLEAKLOG) && + cp->mc_slab_log != NULL) + (*cp->mc_slab_log)(num, *top, TRUE); + if (cp->mc_flags & MCF_DEBUG) goto debug_alloc; @@ -534,11 +572,14 @@ retry_alloc: } } + if (!(cp->mc_flags & MCF_NOLEAKLOG) && cp->mc_slab_log != NULL) + (*cp->mc_slab_log)((num - need), *top, TRUE); + if (!(cp->mc_flags & MCF_DEBUG)) return (num - need); debug_alloc: - if (cp->mc_flags & MCF_VERIFY) { + if (cp->mc_flags & MCF_DEBUG) { mcache_obj_t **o = top; unsigned int n; @@ -561,7 +602,7 @@ debug_alloc: } /* Invoke the slab layer audit callback if auditing is enabled */ - if ((cp->mc_flags & MCF_AUDIT) && cp->mc_slab_audit != NULL) + if ((cp->mc_flags & MCF_DEBUG) && cp->mc_slab_audit != NULL) (*cp->mc_slab_audit)(cp->mc_private, *top, TRUE); return (num - need); @@ -631,11 +672,10 @@ mcache_purge(void *arg) lck_mtx_lock_spin(&cp->mc_sync_lock); cp->mc_enable_cnt++; lck_mtx_unlock(&cp->mc_sync_lock); - } __private_extern__ boolean_t -mcache_purge_cache(mcache_t *cp) +mcache_purge_cache(mcache_t *cp, boolean_t async) { /* * Purging a cache that has no per-CPU caches or is already @@ -652,7 +692,10 @@ mcache_purge_cache(mcache_t *cp) cp->mc_purge_cnt++; lck_mtx_unlock(&cp->mc_sync_lock); - mcache_dispatch(mcache_purge, cp); + if (async) + mcache_dispatch(mcache_purge, cp); + else + mcache_purge(cp); return (TRUE); } @@ -678,8 +721,11 @@ mcache_free_ext(mcache_t *cp, mcache_obj_t *list) mcache_obj_t *nlist; mcache_bkt_t *bkt; + if (!(cp->mc_flags & MCF_NOLEAKLOG) && cp->mc_slab_log != NULL) + (*cp->mc_slab_log)(0, list, FALSE); + /* Invoke the slab layer audit callback if auditing is enabled */ - if ((cp->mc_flags & MCF_AUDIT) && cp->mc_slab_audit != NULL) + if ((cp->mc_flags & MCF_DEBUG) && cp->mc_slab_audit != NULL) (*cp->mc_slab_audit)(cp->mc_private, list, FALSE); MCACHE_LOCK(&ccp->cc_lock); @@ -899,7 +945,7 @@ mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) * the nearest 64-bit multiply; this is because we use * 64-bit memory access to set/check the pattern. */ - if (flags & MCF_AUDIT) { + if (flags & MCF_DEBUG) { VERIFY(((intptr_t)base + rsize) <= ((intptr_t)buf + cp->mc_chunksize)); mcache_set_pattern(MCACHE_FREE_PATTERN, base, rsize); @@ -958,7 +1004,7 @@ mcache_slab_free(void *arg, mcache_obj_t *list, __unused boolean_t purged) /* Get the original address since we're about to free it */ pbuf = (void **)((intptr_t)base - sizeof (void *)); - if (flags & MCF_AUDIT) { + if (flags & MCF_DEBUG) { VERIFY(((intptr_t)base + rsize) <= ((intptr_t)*pbuf + cp->mc_chunksize)); mcache_audit_free_verify(NULL, base, offset, rsize); @@ -1156,7 +1202,7 @@ mcache_bkt_destroy(mcache_t *cp, mcache_bkttype_t *btp, mcache_bkt_t *bkt, if (nobjs > 0) { mcache_obj_t *top = bkt->bkt_obj[nobjs - 1]; - if (cp->mc_flags & MCF_VERIFY) { + if (cp->mc_flags & MCF_DEBUG) { mcache_obj_t *o = top; int cnt = 0; @@ -1222,7 +1268,8 @@ mcache_bkt_ws_reap(mcache_t *cp) } static void -mcache_reap_timeout(void *arg) +mcache_reap_timeout(thread_call_param_t dummy __unused, + thread_call_param_t arg) { volatile UInt32 *flag = arg; @@ -1234,7 +1281,14 @@ mcache_reap_timeout(void *arg) static void mcache_reap_done(void *flag) { - timeout(mcache_reap_timeout, flag, mcache_reap_interval); + uint64_t deadline, leeway; + + clock_interval_to_deadline(mcache_reap_interval, NSEC_PER_SEC, + &deadline); + clock_interval_to_absolutetime_interval(mcache_reap_interval_leeway, + NSEC_PER_SEC, &leeway); + thread_call_enter_delayed_with_leeway(mcache_reap_tcall, flag, + deadline, leeway, THREAD_CALL_DELAY_LEEWAY); } static void @@ -1359,14 +1413,22 @@ mcache_cache_enable(void *arg) static void mcache_update_timeout(__unused void *arg) { - timeout(mcache_update, NULL, mcache_reap_interval); + uint64_t deadline, leeway; + + clock_interval_to_deadline(mcache_reap_interval, NSEC_PER_SEC, + &deadline); + clock_interval_to_absolutetime_interval(mcache_reap_interval_leeway, + NSEC_PER_SEC, &leeway); + thread_call_enter_delayed_with_leeway(mcache_update_tcall, NULL, + deadline, leeway, THREAD_CALL_DELAY_LEEWAY); } static void -mcache_update(__unused void *arg) +mcache_update(thread_call_param_t arg __unused, + thread_call_param_t dummy __unused) { mcache_applyall(mcache_cache_update); - mcache_dispatch(mcache_update_timeout, NULL); + mcache_update_timeout(NULL); } static void @@ -1389,22 +1451,41 @@ mcache_dispatch(void (*func)(void *), void *arg) } __private_extern__ void -mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp) +mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp, + struct timeval *base_ts) { + struct timeval now, base = { 0, 0 }; + void *stack[MCACHE_STACK_DEPTH + 1]; + struct mca_trn *transaction; + + transaction = &mca->mca_trns[mca->mca_next_trn]; + mca->mca_addr = addr; mca->mca_cache = cp; - mca->mca_pthread = mca->mca_thread; - mca->mca_thread = current_thread(); - bcopy(mca->mca_stack, mca->mca_pstack, sizeof (mca->mca_pstack)); - mca->mca_pdepth = mca->mca_depth; - bzero(mca->mca_stack, sizeof (mca->mca_stack)); - mca->mca_depth = OSBacktrace(mca->mca_stack, MCACHE_STACK_DEPTH); + + transaction->mca_thread = current_thread(); + + bzero(stack, sizeof (stack)); + transaction->mca_depth = OSBacktrace(stack, MCACHE_STACK_DEPTH + 1) - 1; + bcopy(&stack[1], transaction->mca_stack, + sizeof (transaction->mca_stack)); + + microuptime(&now); + if (base_ts != NULL) + base = *base_ts; + /* tstamp is in ms relative to base_ts */ + transaction->mca_tstamp = ((now.tv_usec - base.tv_usec) / 1000); + if ((now.tv_sec - base.tv_sec) > 0) + transaction->mca_tstamp += ((now.tv_sec - base.tv_sec) * 1000); + + mca->mca_next_trn = + (mca->mca_next_trn + 1) % mca_trn_max; } __private_extern__ void mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size) { - u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size); + u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size)); u_int64_t *buf = (u_int64_t *)buf_arg; VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t))); @@ -1417,7 +1498,7 @@ mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size) __private_extern__ void * mcache_verify_pattern(u_int64_t pattern, void *buf_arg, size_t size) { - u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size); + u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size)); u_int64_t *buf; VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t))); @@ -1434,7 +1515,7 @@ __private_extern__ void * mcache_verify_set_pattern(u_int64_t old, u_int64_t new, void *buf_arg, size_t size) { - u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size); + u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size)); u_int64_t *buf; VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t))); @@ -1501,6 +1582,26 @@ mcache_audit_free_verify_set(mcache_audit_t *mca, void *base, size_t offset, #undef panic +#define DUMP_TRN_FMT() \ + "%s transaction thread %p saved PC stack (%d deep):\n" \ + "\t%p, %p, %p, %p, %p, %p, %p, %p\n" \ + "\t%p, %p, %p, %p, %p, %p, %p, %p\n" + +#define DUMP_TRN_FIELDS(s, x) \ + s, \ + mca->mca_trns[x].mca_thread, mca->mca_trns[x].mca_depth, \ + mca->mca_trns[x].mca_stack[0], mca->mca_trns[x].mca_stack[1], \ + mca->mca_trns[x].mca_stack[2], mca->mca_trns[x].mca_stack[3], \ + mca->mca_trns[x].mca_stack[4], mca->mca_trns[x].mca_stack[5], \ + mca->mca_trns[x].mca_stack[6], mca->mca_trns[x].mca_stack[7], \ + mca->mca_trns[x].mca_stack[8], mca->mca_trns[x].mca_stack[9], \ + mca->mca_trns[x].mca_stack[10], mca->mca_trns[x].mca_stack[11], \ + mca->mca_trns[x].mca_stack[12], mca->mca_trns[x].mca_stack[13], \ + mca->mca_trns[x].mca_stack[14], mca->mca_trns[x].mca_stack[15] + +#define MCA_TRN_LAST ((mca->mca_next_trn + mca_trn_max) % mca_trn_max) +#define MCA_TRN_PREV ((mca->mca_next_trn + mca_trn_max - 1) % mca_trn_max) + __private_extern__ char * mcache_dump_mca(mcache_audit_t *mca) { @@ -1508,29 +1609,16 @@ mcache_dump_mca(mcache_audit_t *mca) return (NULL); snprintf(mca_dump_buf, DUMP_MCA_BUF_SIZE, - "mca %p: addr %p, cache %p (%s)\n" - "last transaction; thread %p, saved PC stack (%d deep):\n" - "\t%p, %p, %p, %p, %p, %p, %p, %p\n" - "\t%p, %p, %p, %p, %p, %p, %p, %p\n" - "previous transaction; thread %p, saved PC stack (%d deep):\n" - "\t%p, %p, %p, %p, %p, %p, %p, %p\n" - "\t%p, %p, %p, %p, %p, %p, %p, %p\n", + "mca %p: addr %p, cache %p (%s) nxttrn %d\n" + DUMP_TRN_FMT() + DUMP_TRN_FMT(), + mca, mca->mca_addr, mca->mca_cache, mca->mca_cache ? mca->mca_cache->mc_name : "?", - mca->mca_thread, mca->mca_depth, - mca->mca_stack[0], mca->mca_stack[1], mca->mca_stack[2], - mca->mca_stack[3], mca->mca_stack[4], mca->mca_stack[5], - mca->mca_stack[6], mca->mca_stack[7], mca->mca_stack[8], - mca->mca_stack[9], mca->mca_stack[10], mca->mca_stack[11], - mca->mca_stack[12], mca->mca_stack[13], mca->mca_stack[14], - mca->mca_stack[15], - mca->mca_pthread, mca->mca_pdepth, - mca->mca_pstack[0], mca->mca_pstack[1], mca->mca_pstack[2], - mca->mca_pstack[3], mca->mca_pstack[4], mca->mca_pstack[5], - mca->mca_pstack[6], mca->mca_pstack[7], mca->mca_pstack[8], - mca->mca_pstack[9], mca->mca_pstack[10], mca->mca_pstack[11], - mca->mca_pstack[12], mca->mca_pstack[13], mca->mca_pstack[14], - mca->mca_pstack[15]); + mca->mca_next_trn, + + DUMP_TRN_FIELDS("last", MCA_TRN_LAST), + DUMP_TRN_FIELDS("previous", MCA_TRN_PREV)); return (mca_dump_buf); }