/*
- * Copyright (c) 2006-2014 Apple Inc. All rights reserved.
+ * Copyright (c) 2006-2020 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#define MCACHE_UNLOCK(l) lck_mtx_unlock(l)
#define MCACHE_LOCK_TRY(l) lck_mtx_try_lock(l)
-static int ncpu;
+static unsigned int ncpu;
static unsigned int cache_line_size;
static lck_mtx_t *mcache_llock;
static struct thread *mcache_llock_owner;
static void mcache_slab_free(void *, mcache_obj_t *, boolean_t);
static void mcache_slab_audit(void *, mcache_obj_t *, boolean_t);
static void mcache_cpu_refill(mcache_cpu_t *, mcache_bkt_t *, int);
-static mcache_bkt_t *mcache_bkt_alloc(mcache_t *, mcache_bktlist_t *,
- mcache_bkttype_t **);
+static mcache_bkt_t *mcache_bkt_alloc(mcache_t *, mcache_bktlist_t *);
static void mcache_bkt_free(mcache_t *, mcache_bktlist_t *, mcache_bkt_t *);
static void mcache_cache_bkt_enable(mcache_t *);
static void mcache_bkt_purge(mcache_t *);
-static void mcache_bkt_destroy(mcache_t *, mcache_bkttype_t *,
- mcache_bkt_t *, int);
+static void mcache_bkt_destroy(mcache_t *, mcache_bkt_t *, int);
static void mcache_bkt_ws_update(mcache_t *);
static void mcache_bkt_ws_zero(mcache_t *);
static void mcache_bkt_ws_reap(mcache_t *);
VERIFY(mca_trn_max >= 2);
- ncpu = ml_get_max_cpus();
+ ncpu = ml_wait_max_cpus();
(void) mcache_cache_line_size(); /* prime it */
mcache_llock_grp_attr = lck_grp_attr_alloc_init();
mcache_update_tcall = thread_call_allocate(mcache_update, NULL);
if (mcache_reap_tcall == NULL || mcache_update_tcall == NULL) {
panic("mcache_init: thread_call_allocate failed");
+ /* NOTREACHED */
+ __builtin_unreachable();
}
- mcache_zone = zinit(MCACHE_ALLOC_SIZE, 256 * MCACHE_ALLOC_SIZE,
- PAGE_SIZE, "mcache");
- if (mcache_zone == NULL) {
- panic("mcache_init: failed to allocate mcache zone\n");
- }
- zone_change(mcache_zone, Z_CALLERACCT, FALSE);
+ mcache_zone = zone_create("mcache", MCACHE_ALLOC_SIZE, ZC_DESTRUCTIBLE);
LIST_INIT(&mcache_head);
if (cache_line_size == 0) {
ml_cpu_info_t cpu_info;
ml_cpu_get_info(&cpu_info);
- cache_line_size = cpu_info.cache_line_size;
+ cache_line_size = (unsigned int)cpu_info.cache_line_size;
}
return cache_line_size;
}
mcache_t *cp = NULL;
size_t chunksize;
void *buf, **pbuf;
- int c;
+ unsigned int c;
char lck_name[64];
/* If auditing is on and print buffer is NULL, allocate it now */
if ((align & (align - 1)) != 0) {
panic("mcache_create: bad alignment %lu", align);
+ /* NOTREACHED */
+ __builtin_unreachable();
}
cp->mc_align = align;
VERIFY(align != 0 && (align % MCACHE_ALIGN) == 0);
chunksize += sizeof(uint64_t) + align;
chunksize = P2ROUNDUP(chunksize, align);
- if ((cp->mc_slab_zone = zinit(chunksize, 64 * 1024 * ncpu,
- PAGE_SIZE, cp->mc_name)) == NULL) {
- goto fail;
- }
- zone_change(cp->mc_slab_zone, Z_EXPAND, TRUE);
+ cp->mc_slab_zone = zone_create(cp->mc_name, chunksize, ZC_DESTRUCTIBLE);
}
cp->mc_chunksize = chunksize;
* bucket from the bucket layer. Upon success, refill this
* CPU and place any empty bucket into the empty list.
*/
- bkt = mcache_bkt_alloc(cp, &cp->mc_full, NULL);
+ bkt = mcache_bkt_alloc(cp, &cp->mc_full);
if (bkt != NULL) {
if (ccp->cc_pfilled != NULL) {
mcache_bkt_free(cp, &cp->mc_empty,
panic("mcache_alloc_ext: %s cp %p corrupted list "
"(got %d actual %d)\n", cp->mc_name,
(void *)cp, num - need, n);
+ /* NOTREACHED */
+ __builtin_unreachable();
}
}
* bucket from the bucket layer. Upon success, empty this
* CPU and place any full bucket into the full list.
*/
- bkt = mcache_bkt_alloc(cp, &cp->mc_empty, &btp);
+ bkt = mcache_bkt_alloc(cp, &cp->mc_empty);
if (bkt != NULL) {
if (ccp->cc_pfilled != NULL) {
mcache_bkt_free(cp, &cp->mc_full,
mcache_cpu_refill(ccp, bkt, 0);
continue;
}
+ btp = cp->cache_bkttype;
/*
* We need an empty bucket to put our freed objects into
continue;
}
+ /*
+ * Store it in the bucket object since we'll
+ * need to refer to it during bucket destroy;
+ * we can't safely refer to cache_bkttype as
+ * the bucket lock may not be acquired then.
+ */
+ bkt->bkt_type = btp;
+
/*
* We have an empty bucket of the right size;
* add it to the bucket layer and try again.
* Allocate a bucket from the bucket layer.
*/
static mcache_bkt_t *
-mcache_bkt_alloc(mcache_t *cp, mcache_bktlist_t *blp, mcache_bkttype_t **btp)
+mcache_bkt_alloc(mcache_t *cp, mcache_bktlist_t *blp)
{
mcache_bkt_t *bkt;
blp->bl_alloc++;
}
- if (btp != NULL) {
- *btp = cp->cache_bkttype;
- }
-
MCACHE_UNLOCK(&cp->mc_bkt_lock);
return bkt;
mcache_cache_bkt_enable(mcache_t *cp)
{
mcache_cpu_t *ccp;
- int cpu;
+ unsigned int cpu;
if (cp->mc_flags & MCF_NOCPUCACHE) {
return;
{
mcache_cpu_t *ccp;
mcache_bkt_t *bp, *pbp;
- mcache_bkttype_t *btp;
- int cpu, objs, pobjs;
+ int objs, pobjs;
+ unsigned int cpu;
for (cpu = 0; cpu < ncpu; cpu++) {
ccp = &cp->mc_cpu[cpu];
MCACHE_LOCK(&ccp->cc_lock);
- btp = cp->cache_bkttype;
bp = ccp->cc_filled;
pbp = ccp->cc_pfilled;
objs = ccp->cc_objs;
MCACHE_UNLOCK(&ccp->cc_lock);
if (bp != NULL) {
- mcache_bkt_destroy(cp, btp, bp, objs);
+ mcache_bkt_destroy(cp, bp, objs);
}
if (pbp != NULL) {
- mcache_bkt_destroy(cp, btp, pbp, pobjs);
+ mcache_bkt_destroy(cp, pbp, pobjs);
}
}
* and also free the bucket itself.
*/
static void
-mcache_bkt_destroy(mcache_t *cp, mcache_bkttype_t *btp, mcache_bkt_t *bkt,
- int nobjs)
+mcache_bkt_destroy(mcache_t *cp, mcache_bkt_t *bkt, int nobjs)
{
if (nobjs > 0) {
mcache_obj_t *top = bkt->bkt_obj[nobjs - 1];
"list in bkt %p (nobjs %d actual %d)\n",
cp->mc_name, (void *)cp, (void *)bkt,
nobjs, cnt);
+ /* NOTREACHED */
+ __builtin_unreachable();
}
}
(*cp->mc_slab_free)(cp->mc_private, top,
(cp->mc_flags & MCF_DEBUG) || cp->mc_purge_cnt);
}
- mcache_free(btp->bt_cache, bkt);
+ mcache_free(bkt->bkt_type->bt_cache, bkt);
}
/*
{
long reap;
mcache_bkt_t *bkt;
- mcache_bkttype_t *btp;
reap = MIN(cp->mc_full.bl_reaplimit, cp->mc_full.bl_min);
while (reap-- &&
- (bkt = mcache_bkt_alloc(cp, &cp->mc_full, &btp)) != NULL) {
- mcache_bkt_destroy(cp, btp, bkt, btp->bt_bktsize);
+ (bkt = mcache_bkt_alloc(cp, &cp->mc_full)) != NULL) {
+ mcache_bkt_destroy(cp, bkt, bkt->bkt_type->bt_bktsize);
}
reap = MIN(cp->mc_empty.bl_reaplimit, cp->mc_empty.bl_min);
while (reap-- &&
- (bkt = mcache_bkt_alloc(cp, &cp->mc_empty, &btp)) != NULL) {
- mcache_bkt_destroy(cp, btp, bkt, 0);
+ (bkt = mcache_bkt_alloc(cp, &cp->mc_empty)) != NULL) {
+ mcache_bkt_destroy(cp, bkt, 0);
}
}
mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp,
struct timeval *base_ts)
{
- struct timeval now, base = { 0, 0 };
+ struct timeval now, base = { .tv_sec = 0, .tv_usec = 0 };
void *stack[MCACHE_STACK_DEPTH + 1];
struct mca_trn *transaction;
transaction->mca_thread = current_thread();
bzero(stack, sizeof(stack));
- transaction->mca_depth = OSBacktrace(stack, MCACHE_STACK_DEPTH + 1) - 1;
+ transaction->mca_depth = (uint16_t)OSBacktrace(stack, MCACHE_STACK_DEPTH + 1) - 1;
bcopy(&stack[1], transaction->mca_stack,
sizeof(transaction->mca_stack));
(mca->mca_next_trn + 1) % mca_trn_max;
}
-__private_extern__ void
+/*
+ * N.B.: mcache_set_pattern(), mcache_verify_pattern() and
+ * mcache_verify_set_pattern() are marked as noinline to prevent the
+ * compiler from aliasing pointers when they are inlined inside the callers
+ * (e.g. mcache_audit_free_verify_set()) which would be undefined behavior.
+ */
+__private_extern__ OS_NOINLINE void
mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size)
{
u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));
}
}
-__private_extern__ void *
+__private_extern__ OS_NOINLINE void *
mcache_verify_pattern(u_int64_t pattern, void *buf_arg, size_t size)
{
u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));
return NULL;
}
-__private_extern__ void *
+OS_NOINLINE static void *
mcache_verify_set_pattern(u_int64_t old, u_int64_t new, void *buf_arg,
size_t size)
{
"offset 0x%lx (0x%llx instead of 0x%llx)\n", addr,
offset, got, expected);
/* NOTREACHED */
+ __builtin_unreachable();
}
panic("mcache_audit: buffer %p modified after free at offset 0x%lx "
"(0x%llx instead of 0x%llx)\n%s\n",
addr, offset, got, expected, mcache_dump_mca(mca));
/* NOTREACHED */
+ __builtin_unreachable();
}
+__attribute__((noinline, cold, not_tail_called, noreturn))
__private_extern__ int
assfail(const char *a, const char *f, int l)
{
panic("assertion failed: %s, file: %s, line: %d", a, f, l);
- return 0;
+ /* NOTREACHED */
+ __builtin_unreachable();
}
projects / apple / xnu.git / blobdiff