xnu-7195.50.7.100.1.tar.gz

[apple/xnu.git] / bsd / kern / mcache.c

diff --git a/bsd/kern/mcache.c b/bsd/kern/mcache.c
index a2263417d41c1209be3f17d3b4ec25848cfbc34b..76b4c601e5cc2b524666065aad780c189a4ff963 100644 (file)
--- a/bsd/kern/mcache.c
+++ b/bsd/kern/mcache.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2006-2014 Apple Inc. All rights reserved.
+ * Copyright (c) 2006-2020 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *


@@ -99,7 +99,7 @@
 #define MCACHE_UNLOCK(l)        lck_mtx_unlock(l)
 #define MCACHE_LOCK_TRY(l)      lck_mtx_try_lock(l)
 
 #define MCACHE_UNLOCK(l)        lck_mtx_unlock(l)
 #define MCACHE_LOCK_TRY(l)      lck_mtx_try_lock(l)
 

-static int ncpu;
+static unsigned int ncpu;

 static unsigned int cache_line_size;
 static lck_mtx_t *mcache_llock;
 static struct thread *mcache_llock_owner;
 static unsigned int cache_line_size;
 static lck_mtx_t *mcache_llock;
 static struct thread *mcache_llock_owner;


@@ -146,13 +146,11 @@ static unsigned int mcache_slab_alloc(void *, mcache_obj_t ***,
 static void mcache_slab_free(void *, mcache_obj_t *, boolean_t);
 static void mcache_slab_audit(void *, mcache_obj_t *, boolean_t);
 static void mcache_cpu_refill(mcache_cpu_t *, mcache_bkt_t *, int);
 static void mcache_slab_free(void *, mcache_obj_t *, boolean_t);
 static void mcache_slab_audit(void *, mcache_obj_t *, boolean_t);
 static void mcache_cpu_refill(mcache_cpu_t *, mcache_bkt_t *, int);

-static mcache_bkt_t *mcache_bkt_alloc(mcache_t *, mcache_bktlist_t *,
-    mcache_bkttype_t **);
+static mcache_bkt_t *mcache_bkt_alloc(mcache_t *, mcache_bktlist_t *);

 static void mcache_bkt_free(mcache_t *, mcache_bktlist_t *, mcache_bkt_t *);
 static void mcache_cache_bkt_enable(mcache_t *);
 static void mcache_bkt_purge(mcache_t *);
 static void mcache_bkt_free(mcache_t *, mcache_bktlist_t *, mcache_bkt_t *);
 static void mcache_cache_bkt_enable(mcache_t *);
 static void mcache_bkt_purge(mcache_t *);

-static void mcache_bkt_destroy(mcache_t *, mcache_bkttype_t *,
-    mcache_bkt_t *, int);
+static void mcache_bkt_destroy(mcache_t *, mcache_bkt_t *, int);

 static void mcache_bkt_ws_update(mcache_t *);
 static void mcache_bkt_ws_zero(mcache_t *);
 static void mcache_bkt_ws_reap(mcache_t *);
 static void mcache_bkt_ws_update(mcache_t *);
 static void mcache_bkt_ws_zero(mcache_t *);
 static void mcache_bkt_ws_reap(mcache_t *);


@@ -188,7 +186,7 @@ mcache_init(void)
 
        VERIFY(mca_trn_max >= 2);
 
 
        VERIFY(mca_trn_max >= 2);
 

-       ncpu = ml_get_max_cpus();
+       ncpu = ml_wait_max_cpus();

        (void) mcache_cache_line_size();        /* prime it */
 
        mcache_llock_grp_attr = lck_grp_attr_alloc_init();
        (void) mcache_cache_line_size();        /* prime it */
 
        mcache_llock_grp_attr = lck_grp_attr_alloc_init();


@@ -201,14 +199,11 @@ mcache_init(void)
        mcache_update_tcall = thread_call_allocate(mcache_update, NULL);
        if (mcache_reap_tcall == NULL || mcache_update_tcall == NULL) {
                panic("mcache_init: thread_call_allocate failed");
        mcache_update_tcall = thread_call_allocate(mcache_update, NULL);
        if (mcache_reap_tcall == NULL || mcache_update_tcall == NULL) {
                panic("mcache_init: thread_call_allocate failed");

+               /* NOTREACHED */
+               __builtin_unreachable();

        }
 
        }
 

-       mcache_zone = zinit(MCACHE_ALLOC_SIZE, 256 * MCACHE_ALLOC_SIZE,
-           PAGE_SIZE, "mcache");
-       if (mcache_zone == NULL) {
-               panic("mcache_init: failed to allocate mcache zone\n");
-       }
-       zone_change(mcache_zone, Z_CALLERACCT, FALSE);
+       mcache_zone = zone_create("mcache", MCACHE_ALLOC_SIZE, ZC_DESTRUCTIBLE);

 
        LIST_INIT(&mcache_head);
 
 
        LIST_INIT(&mcache_head);
 


@@ -251,7 +246,7 @@ mcache_cache_line_size(void)
        if (cache_line_size == 0) {
                ml_cpu_info_t cpu_info;
                ml_cpu_get_info(&cpu_info);
        if (cache_line_size == 0) {
                ml_cpu_info_t cpu_info;
                ml_cpu_get_info(&cpu_info);

-               cache_line_size = cpu_info.cache_line_size;
+               cache_line_size = (unsigned int)cpu_info.cache_line_size;

        }
        return cache_line_size;
 }
        }
        return cache_line_size;
 }


@@ -298,7 +293,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
        mcache_t *cp = NULL;
        size_t chunksize;
        void *buf, **pbuf;
        mcache_t *cp = NULL;
        size_t chunksize;
        void *buf, **pbuf;

-       int c;
+       unsigned int c;

        char lck_name[64];
 
        /* If auditing is on and print buffer is NULL, allocate it now */
        char lck_name[64];
 
        /* If auditing is on and print buffer is NULL, allocate it now */


@@ -346,6 +341,8 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
 
        if ((align & (align - 1)) != 0) {
                panic("mcache_create: bad alignment %lu", align);
 
        if ((align & (align - 1)) != 0) {
                panic("mcache_create: bad alignment %lu", align);

+               /* NOTREACHED */
+               __builtin_unreachable();

        }
 
        cp->mc_align = align;
        }
 
        cp->mc_align = align;


@@ -378,11 +375,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
                VERIFY(align != 0 && (align % MCACHE_ALIGN) == 0);
                chunksize += sizeof(uint64_t) + align;
                chunksize = P2ROUNDUP(chunksize, align);
                VERIFY(align != 0 && (align % MCACHE_ALIGN) == 0);
                chunksize += sizeof(uint64_t) + align;
                chunksize = P2ROUNDUP(chunksize, align);

-               if ((cp->mc_slab_zone = zinit(chunksize, 64 * 1024 * ncpu,
-                   PAGE_SIZE, cp->mc_name)) == NULL) {
-                       goto fail;
-               }
-               zone_change(cp->mc_slab_zone, Z_EXPAND, TRUE);
+               cp->mc_slab_zone = zone_create(cp->mc_name, chunksize, ZC_DESTRUCTIBLE);

        }
        cp->mc_chunksize = chunksize;
 
        }
        cp->mc_chunksize = chunksize;
 


@@ -548,7 +541,7 @@ retry_alloc:
                 * bucket from the bucket layer.  Upon success, refill this
                 * CPU and place any empty bucket into the empty list.
                 */
                 * bucket from the bucket layer.  Upon success, refill this
                 * CPU and place any empty bucket into the empty list.
                 */

-               bkt = mcache_bkt_alloc(cp, &cp->mc_full, NULL);
+               bkt = mcache_bkt_alloc(cp, &cp->mc_full);

                if (bkt != NULL) {
                        if (ccp->cc_pfilled != NULL) {
                                mcache_bkt_free(cp, &cp->mc_empty,
                if (bkt != NULL) {
                        if (ccp->cc_pfilled != NULL) {
                                mcache_bkt_free(cp, &cp->mc_empty,


@@ -616,6 +609,8 @@ debug_alloc:
                        panic("mcache_alloc_ext: %s cp %p corrupted list "
                            "(got %d actual %d)\n", cp->mc_name,
                            (void *)cp, num - need, n);
                        panic("mcache_alloc_ext: %s cp %p corrupted list "
                            "(got %d actual %d)\n", cp->mc_name,
                            (void *)cp, num - need, n);

+                       /* NOTREACHED */
+                       __builtin_unreachable();

                }
        }
 
                }
        }
 


@@ -810,7 +805,7 @@ mcache_free_ext(mcache_t *cp, mcache_obj_t *list)
                 * bucket from the bucket layer.  Upon success, empty this
                 * CPU and place any full bucket into the full list.
                 */
                 * bucket from the bucket layer.  Upon success, empty this
                 * CPU and place any full bucket into the full list.
                 */

-               bkt = mcache_bkt_alloc(cp, &cp->mc_empty, &btp);
+               bkt = mcache_bkt_alloc(cp, &cp->mc_empty);

                if (bkt != NULL) {
                        if (ccp->cc_pfilled != NULL) {
                                mcache_bkt_free(cp, &cp->mc_full,
                if (bkt != NULL) {
                        if (ccp->cc_pfilled != NULL) {
                                mcache_bkt_free(cp, &cp->mc_full,


@@ -819,6 +814,7 @@ mcache_free_ext(mcache_t *cp, mcache_obj_t *list)
                        mcache_cpu_refill(ccp, bkt, 0);
                        continue;
                }
                        mcache_cpu_refill(ccp, bkt, 0);
                        continue;
                }

+               btp = cp->cache_bkttype;

 
                /*
                 * We need an empty bucket to put our freed objects into
 
                /*
                 * We need an empty bucket to put our freed objects into


@@ -844,6 +840,14 @@ mcache_free_ext(mcache_t *cp, mcache_obj_t *list)
                                continue;
                        }
 
                                continue;
                        }
 

+                       /*
+                        * Store it in the bucket object since we'll
+                        * need to refer to it during bucket destroy;
+                        * we can't safely refer to cache_bkttype as
+                        * the bucket lock may not be acquired then.
+                        */
+                       bkt->bkt_type = btp;
+

                        /*
                         * We have an empty bucket of the right size;
                         * add it to the bucket layer and try again.
                        /*
                         * We have an empty bucket of the right size;
                         * add it to the bucket layer and try again.


@@ -1082,7 +1086,7 @@ mcache_cpu_refill(mcache_cpu_t *ccp, mcache_bkt_t *bkt, int objs)
  * Allocate a bucket from the bucket layer.
  */
 static mcache_bkt_t *
  * Allocate a bucket from the bucket layer.
  */
 static mcache_bkt_t *

-mcache_bkt_alloc(mcache_t *cp, mcache_bktlist_t *blp, mcache_bkttype_t **btp)
+mcache_bkt_alloc(mcache_t *cp, mcache_bktlist_t *blp)

 {
        mcache_bkt_t *bkt;
 
 {
        mcache_bkt_t *bkt;
 


@@ -1104,10 +1108,6 @@ mcache_bkt_alloc(mcache_t *cp, mcache_bktlist_t *blp, mcache_bkttype_t **btp)
                blp->bl_alloc++;
        }
 
                blp->bl_alloc++;
        }
 

-       if (btp != NULL) {
-               *btp = cp->cache_bkttype;
-       }
-

        MCACHE_UNLOCK(&cp->mc_bkt_lock);
 
        return bkt;
        MCACHE_UNLOCK(&cp->mc_bkt_lock);
 
        return bkt;


@@ -1135,7 +1135,7 @@ static void
 mcache_cache_bkt_enable(mcache_t *cp)
 {
        mcache_cpu_t *ccp;
 mcache_cache_bkt_enable(mcache_t *cp)
 {
        mcache_cpu_t *ccp;

-       int cpu;
+       unsigned int cpu;

 
        if (cp->mc_flags & MCF_NOCPUCACHE) {
                return;
 
        if (cp->mc_flags & MCF_NOCPUCACHE) {
                return;


@@ -1157,15 +1157,14 @@ mcache_bkt_purge(mcache_t *cp)
 {
        mcache_cpu_t *ccp;
        mcache_bkt_t *bp, *pbp;
 {
        mcache_cpu_t *ccp;
        mcache_bkt_t *bp, *pbp;

-       mcache_bkttype_t *btp;
-       int cpu, objs, pobjs;
+       int objs, pobjs;
+       unsigned int cpu;

 
        for (cpu = 0; cpu < ncpu; cpu++) {
                ccp = &cp->mc_cpu[cpu];
 
                MCACHE_LOCK(&ccp->cc_lock);
 
 
        for (cpu = 0; cpu < ncpu; cpu++) {
                ccp = &cp->mc_cpu[cpu];
 
                MCACHE_LOCK(&ccp->cc_lock);
 

-               btp = cp->cache_bkttype;

                bp = ccp->cc_filled;
                pbp = ccp->cc_pfilled;
                objs = ccp->cc_objs;
                bp = ccp->cc_filled;
                pbp = ccp->cc_pfilled;
                objs = ccp->cc_objs;


@@ -1179,10 +1178,10 @@ mcache_bkt_purge(mcache_t *cp)
                MCACHE_UNLOCK(&ccp->cc_lock);
 
                if (bp != NULL) {
                MCACHE_UNLOCK(&ccp->cc_lock);
 
                if (bp != NULL) {

-                       mcache_bkt_destroy(cp, btp, bp, objs);
+                       mcache_bkt_destroy(cp, bp, objs);

                }
                if (pbp != NULL) {
                }
                if (pbp != NULL) {

-                       mcache_bkt_destroy(cp, btp, pbp, pobjs);
+                       mcache_bkt_destroy(cp, pbp, pobjs);

                }
        }
 
                }
        }
 


@@ -1195,8 +1194,7 @@ mcache_bkt_purge(mcache_t *cp)
  * and also free the bucket itself.
  */
 static void
  * and also free the bucket itself.
  */
 static void

-mcache_bkt_destroy(mcache_t *cp, mcache_bkttype_t *btp, mcache_bkt_t *bkt,
-    int nobjs)
+mcache_bkt_destroy(mcache_t *cp, mcache_bkt_t *bkt, int nobjs)

 {
        if (nobjs > 0) {
                mcache_obj_t *top = bkt->bkt_obj[nobjs - 1];
 {
        if (nobjs > 0) {
                mcache_obj_t *top = bkt->bkt_obj[nobjs - 1];


@@ -1219,6 +1217,8 @@ mcache_bkt_destroy(mcache_t *cp, mcache_bkttype_t *btp, mcache_bkt_t *bkt,
                                    "list in bkt %p (nobjs %d actual %d)\n",
                                    cp->mc_name, (void *)cp, (void *)bkt,
                                    nobjs, cnt);
                                    "list in bkt %p (nobjs %d actual %d)\n",
                                    cp->mc_name, (void *)cp, (void *)bkt,
                                    nobjs, cnt);

+                               /* NOTREACHED */
+                               __builtin_unreachable();

                        }
                }
 
                        }
                }
 


@@ -1226,7 +1226,7 @@ mcache_bkt_destroy(mcache_t *cp, mcache_bkttype_t *btp, mcache_bkt_t *bkt,
                (*cp->mc_slab_free)(cp->mc_private, top,
                    (cp->mc_flags & MCF_DEBUG) || cp->mc_purge_cnt);
        }
                (*cp->mc_slab_free)(cp->mc_private, top,
                    (cp->mc_flags & MCF_DEBUG) || cp->mc_purge_cnt);
        }

-       mcache_free(btp->bt_cache, bkt);
+       mcache_free(bkt->bkt_type->bt_cache, bkt);

 }
 
 /*
 }
 
 /*


@@ -1269,18 +1269,17 @@ mcache_bkt_ws_reap(mcache_t *cp)
 {
        long reap;
        mcache_bkt_t *bkt;
 {
        long reap;
        mcache_bkt_t *bkt;

-       mcache_bkttype_t *btp;

 
        reap = MIN(cp->mc_full.bl_reaplimit, cp->mc_full.bl_min);
        while (reap-- &&
 
        reap = MIN(cp->mc_full.bl_reaplimit, cp->mc_full.bl_min);
        while (reap-- &&

-           (bkt = mcache_bkt_alloc(cp, &cp->mc_full, &btp)) != NULL) {
-               mcache_bkt_destroy(cp, btp, bkt, btp->bt_bktsize);
+           (bkt = mcache_bkt_alloc(cp, &cp->mc_full)) != NULL) {
+               mcache_bkt_destroy(cp, bkt, bkt->bkt_type->bt_bktsize);

        }
 
        reap = MIN(cp->mc_empty.bl_reaplimit, cp->mc_empty.bl_min);
        while (reap-- &&
        }
 
        reap = MIN(cp->mc_empty.bl_reaplimit, cp->mc_empty.bl_min);
        while (reap-- &&

-           (bkt = mcache_bkt_alloc(cp, &cp->mc_empty, &btp)) != NULL) {
-               mcache_bkt_destroy(cp, btp, bkt, 0);
+           (bkt = mcache_bkt_alloc(cp, &cp->mc_empty)) != NULL) {
+               mcache_bkt_destroy(cp, bkt, 0);

        }
 }
 
        }
 }
 


@@ -1487,7 +1486,7 @@ __private_extern__ void
 mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp,
     struct timeval *base_ts)
 {
 mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp,
     struct timeval *base_ts)
 {

-       struct timeval now, base = { 0, 0 };
+       struct timeval now, base = { .tv_sec = 0, .tv_usec = 0 };

        void *stack[MCACHE_STACK_DEPTH + 1];
        struct mca_trn *transaction;
 
        void *stack[MCACHE_STACK_DEPTH + 1];
        struct mca_trn *transaction;
 


@@ -1499,7 +1498,7 @@ mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp,
        transaction->mca_thread = current_thread();
 
        bzero(stack, sizeof(stack));
        transaction->mca_thread = current_thread();
 
        bzero(stack, sizeof(stack));

-       transaction->mca_depth = OSBacktrace(stack, MCACHE_STACK_DEPTH + 1) - 1;
+       transaction->mca_depth = (uint16_t)OSBacktrace(stack, MCACHE_STACK_DEPTH + 1) - 1;

        bcopy(&stack[1], transaction->mca_stack,
            sizeof(transaction->mca_stack));
 
        bcopy(&stack[1], transaction->mca_stack,
            sizeof(transaction->mca_stack));
 


@@ -1517,7 +1516,13 @@ mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp,
            (mca->mca_next_trn + 1) % mca_trn_max;
 }
 
            (mca->mca_next_trn + 1) % mca_trn_max;
 }
 

-__private_extern__ void
+/*
+ * N.B.: mcache_set_pattern(), mcache_verify_pattern() and
+ * mcache_verify_set_pattern() are marked as noinline to prevent the
+ * compiler from aliasing pointers when they are inlined inside the callers
+ * (e.g. mcache_audit_free_verify_set()) which would be undefined behavior.
+ */
+__private_extern__ OS_NOINLINE void

 mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 {
        u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));
 mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 {
        u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));


@@ -1531,7 +1536,7 @@ mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size)
        }
 }
 
        }
 }
 

-__private_extern__ void *
+__private_extern__ OS_NOINLINE void *

 mcache_verify_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 {
        u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));
 mcache_verify_pattern(u_int64_t pattern, void *buf_arg, size_t size)
 {
        u_int64_t *buf_end = (u_int64_t *)((void *)((char *)buf_arg + size));


@@ -1548,7 +1553,7 @@ mcache_verify_pattern(u_int64_t pattern, void *buf_arg, size_t size)
        return NULL;
 }
 
        return NULL;
 }
 

-__private_extern__ void *
+OS_NOINLINE static void *

 mcache_verify_set_pattern(u_int64_t old, u_int64_t new, void *buf_arg,
     size_t size)
 {
 mcache_verify_set_pattern(u_int64_t old, u_int64_t new, void *buf_arg,
     size_t size)
 {


@@ -1670,17 +1675,21 @@ mcache_audit_panic(mcache_audit_t *mca, void *addr, size_t offset,
                    "offset 0x%lx (0x%llx instead of 0x%llx)\n", addr,
                    offset, got, expected);
                /* NOTREACHED */
                    "offset 0x%lx (0x%llx instead of 0x%llx)\n", addr,
                    offset, got, expected);
                /* NOTREACHED */

+               __builtin_unreachable();

        }
 
        panic("mcache_audit: buffer %p modified after free at offset 0x%lx "
            "(0x%llx instead of 0x%llx)\n%s\n",
            addr, offset, got, expected, mcache_dump_mca(mca));
        /* NOTREACHED */
        }
 
        panic("mcache_audit: buffer %p modified after free at offset 0x%lx "
            "(0x%llx instead of 0x%llx)\n%s\n",
            addr, offset, got, expected, mcache_dump_mca(mca));
        /* NOTREACHED */

+       __builtin_unreachable();

 }
 
 }
 

+__attribute__((noinline, cold, not_tail_called, noreturn))

 __private_extern__ int
 assfail(const char *a, const char *f, int l)
 {
        panic("assertion failed: %s, file: %s, line: %d", a, f, l);
 __private_extern__ int
 assfail(const char *a, const char *f, int l)
 {
        panic("assertion failed: %s, file: %s, line: %d", a, f, l);

-       return 0;
+       /* NOTREACHED */
+       __builtin_unreachable();

 }
 }