xnu-7195.101.1.tar.gz

[apple/xnu.git] / bsd / kern / mcache.c

diff --git a/bsd/kern/mcache.c b/bsd/kern/mcache.c
index 76b4c601e5cc2b524666065aad780c189a4ff963..693a144377d05481423bd84fc7f6f02fa1894c2f 100644 (file)
--- a/bsd/kern/mcache.c
+++ b/bsd/kern/mcache.c
@@ -86,13 +86,13 @@
  * caches when memory runs low.
  */
 #define MCACHE_LIST_LOCK() {                            \
  * caches when memory runs low.
  */
 #define MCACHE_LIST_LOCK() {                            \

-       lck_mtx_lock(mcache_llock);                     \
+       lck_mtx_lock(&mcache_llock);                     \

        mcache_llock_owner = current_thread();          \
 }
 
 #define MCACHE_LIST_UNLOCK() {                          \
        mcache_llock_owner = NULL;                      \
        mcache_llock_owner = current_thread();          \
 }
 
 #define MCACHE_LIST_UNLOCK() {                          \
        mcache_llock_owner = NULL;                      \

-       lck_mtx_unlock(mcache_llock);                   \
+       lck_mtx_unlock(&mcache_llock);                   \

 }
 
 #define MCACHE_LOCK(l)          lck_mtx_lock(l)
 }
 
 #define MCACHE_LOCK(l)          lck_mtx_lock(l)


@@ -101,11 +101,9 @@
 
 static unsigned int ncpu;
 static unsigned int cache_line_size;
 
 static unsigned int ncpu;
 static unsigned int cache_line_size;

-static lck_mtx_t *mcache_llock;

 static struct thread *mcache_llock_owner;
 static struct thread *mcache_llock_owner;

-static lck_attr_t *mcache_llock_attr;
-static lck_grp_t *mcache_llock_grp;
-static lck_grp_attr_t *mcache_llock_grp_attr;
+static LCK_GRP_DECLARE(mcache_llock_grp, "mcache.list");
+static LCK_MTX_DECLARE(mcache_llock, &mcache_llock_grp);

 static struct zone *mcache_zone;
 static const uint32_t mcache_reap_interval = 15;
 static const uint32_t mcache_reap_interval_leeway = 2;
 static struct zone *mcache_zone;
 static const uint32_t mcache_reap_interval = 15;
 static const uint32_t mcache_reap_interval_leeway = 2;


@@ -122,9 +120,6 @@ static unsigned int mcache_flags = 0;
 
 int mca_trn_max = MCA_TRN_MAX;
 
 
 int mca_trn_max = MCA_TRN_MAX;
 

-#define DUMP_MCA_BUF_SIZE       512
-static char *mca_dump_buf;
-

 static mcache_bkttype_t mcache_bkttype[] = {
        { 1, 4096, 32768, NULL },
        { 3, 2048, 16384, NULL },
 static mcache_bkttype_t mcache_bkttype[] = {
        { 1, 4096, 32768, NULL },
        { 3, 2048, 16384, NULL },


@@ -140,7 +135,7 @@ static mcache_bkttype_t mcache_bkttype[] = {
 
 static mcache_t *mcache_create_common(const char *, size_t, size_t,
     mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_logfn_t,
 
 static mcache_t *mcache_create_common(const char *, size_t, size_t,
     mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_logfn_t,

-    mcache_notifyfn_t, void *, u_int32_t, int, int);
+    mcache_notifyfn_t, void *, u_int32_t, int);

 static unsigned int mcache_slab_alloc(void *, mcache_obj_t ***,
     unsigned int, int);
 static void mcache_slab_free(void *, mcache_obj_t *, boolean_t);
 static unsigned int mcache_slab_alloc(void *, mcache_obj_t ***,
     unsigned int, int);
 static void mcache_slab_free(void *, mcache_obj_t *, boolean_t);


@@ -189,12 +184,6 @@ mcache_init(void)
        ncpu = ml_wait_max_cpus();
        (void) mcache_cache_line_size();        /* prime it */
 
        ncpu = ml_wait_max_cpus();
        (void) mcache_cache_line_size();        /* prime it */
 

-       mcache_llock_grp_attr = lck_grp_attr_alloc_init();
-       mcache_llock_grp = lck_grp_alloc_init("mcache.list",
-           mcache_llock_grp_attr);
-       mcache_llock_attr = lck_attr_alloc_init();
-       mcache_llock = lck_mtx_alloc_init(mcache_llock_grp, mcache_llock_attr);
-

        mcache_reap_tcall = thread_call_allocate(mcache_reap_timeout, NULL);
        mcache_update_tcall = thread_call_allocate(mcache_update, NULL);
        if (mcache_reap_tcall == NULL || mcache_update_tcall == NULL) {
        mcache_reap_tcall = thread_call_allocate(mcache_reap_timeout, NULL);
        mcache_update_tcall = thread_call_allocate(mcache_update, NULL);
        if (mcache_reap_tcall == NULL || mcache_update_tcall == NULL) {


@@ -258,11 +247,10 @@ mcache_cache_line_size(void)
  */
 __private_extern__ mcache_t *
 mcache_create(const char *name, size_t bufsize, size_t align,
  */
 __private_extern__ mcache_t *
 mcache_create(const char *name, size_t bufsize, size_t align,

-    u_int32_t flags, int wait)
+    u_int32_t flags, int wait __unused)

 {
        return mcache_create_common(name, bufsize, align, mcache_slab_alloc,
 {
        return mcache_create_common(name, bufsize, align, mcache_slab_alloc,

-                  mcache_slab_free, mcache_slab_audit, NULL, NULL, NULL, flags, 1,
-                  wait);
+                  mcache_slab_free, mcache_slab_audit, NULL, NULL, NULL, flags, 1);

 }
 
 /*
 }
 
 /*


@@ -274,10 +262,10 @@ __private_extern__ mcache_t *
 mcache_create_ext(const char *name, size_t bufsize,
     mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,
     mcache_logfn_t logfn, mcache_notifyfn_t notifyfn, void *arg,
 mcache_create_ext(const char *name, size_t bufsize,
     mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,
     mcache_logfn_t logfn, mcache_notifyfn_t notifyfn, void *arg,

-    u_int32_t flags, int wait)
+    u_int32_t flags, int wait __unused)

 {
        return mcache_create_common(name, bufsize, 0, allocfn,
 {
        return mcache_create_common(name, bufsize, 0, allocfn,

-                  freefn, auditfn, logfn, notifyfn, arg, flags, 0, wait);
+                  freefn, auditfn, logfn, notifyfn, arg, flags, 0);

 }
 
 /*
 }
 
 /*


@@ -287,7 +275,7 @@ static mcache_t *
 mcache_create_common(const char *name, size_t bufsize, size_t align,
     mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,
     mcache_logfn_t logfn, mcache_notifyfn_t notifyfn, void *arg,
 mcache_create_common(const char *name, size_t bufsize, size_t align,
     mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,
     mcache_logfn_t logfn, mcache_notifyfn_t notifyfn, void *arg,

-    u_int32_t flags, int need_zone, int wait)
+    u_int32_t flags, int need_zone)

 {
        mcache_bkttype_t *btp;
        mcache_t *cp = NULL;
 {
        mcache_bkttype_t *btp;
        mcache_t *cp = NULL;


@@ -296,23 +284,11 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
        unsigned int c;
        char lck_name[64];
 
        unsigned int c;
        char lck_name[64];
 

-       /* If auditing is on and print buffer is NULL, allocate it now */
-       if ((flags & MCF_DEBUG) && mca_dump_buf == NULL) {
-               int malloc_wait = (wait & MCR_NOSLEEP) ? M_NOWAIT : M_WAITOK;
-               MALLOC(mca_dump_buf, char *, DUMP_MCA_BUF_SIZE, M_TEMP,
-                   malloc_wait | M_ZERO);
-               if (mca_dump_buf == NULL) {
-                       return NULL;
-               }
-       }
-
-       buf = zalloc(mcache_zone);
+       buf = zalloc_flags(mcache_zone, Z_WAITOK | Z_ZERO);

        if (buf == NULL) {
                goto fail;
        }
 
        if (buf == NULL) {
                goto fail;
        }
 

-       bzero(buf, MCACHE_ALLOC_SIZE);
-

        /*
         * In case we didn't get a cache-aligned memory, round it up
         * accordingly.  This is needed in order to get the rest of
        /*
         * In case we didn't get a cache-aligned memory, round it up
         * accordingly.  This is needed in order to get the rest of


@@ -358,10 +334,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
        (void) snprintf(cp->mc_name, sizeof(cp->mc_name), "mcache.%s", name);
 
        (void) snprintf(lck_name, sizeof(lck_name), "%s.cpu", cp->mc_name);
        (void) snprintf(cp->mc_name, sizeof(cp->mc_name), "mcache.%s", name);
 
        (void) snprintf(lck_name, sizeof(lck_name), "%s.cpu", cp->mc_name);

-       cp->mc_cpu_lock_grp_attr = lck_grp_attr_alloc_init();
-       cp->mc_cpu_lock_grp = lck_grp_alloc_init(lck_name,
-           cp->mc_cpu_lock_grp_attr);
-       cp->mc_cpu_lock_attr = lck_attr_alloc_init();
+       cp->mc_cpu_lock_grp = lck_grp_alloc_init(lck_name, LCK_GRP_ATTR_NULL);

 
        /*
         * Allocation chunk size is the object's size plus any extra size
 
        /*
         * Allocation chunk size is the object's size plus any extra size


@@ -383,20 +356,14 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
         * Initialize the bucket layer.
         */
        (void) snprintf(lck_name, sizeof(lck_name), "%s.bkt", cp->mc_name);
         * Initialize the bucket layer.
         */
        (void) snprintf(lck_name, sizeof(lck_name), "%s.bkt", cp->mc_name);

-       cp->mc_bkt_lock_grp_attr = lck_grp_attr_alloc_init();

        cp->mc_bkt_lock_grp = lck_grp_alloc_init(lck_name,
        cp->mc_bkt_lock_grp = lck_grp_alloc_init(lck_name,

-           cp->mc_bkt_lock_grp_attr);
-       cp->mc_bkt_lock_attr = lck_attr_alloc_init();
-       lck_mtx_init(&cp->mc_bkt_lock, cp->mc_bkt_lock_grp,
-           cp->mc_bkt_lock_attr);
+           LCK_GRP_ATTR_NULL);
+       lck_mtx_init(&cp->mc_bkt_lock, cp->mc_bkt_lock_grp, LCK_ATTR_NULL);

 
        (void) snprintf(lck_name, sizeof(lck_name), "%s.sync", cp->mc_name);
 
        (void) snprintf(lck_name, sizeof(lck_name), "%s.sync", cp->mc_name);

-       cp->mc_sync_lock_grp_attr = lck_grp_attr_alloc_init();

        cp->mc_sync_lock_grp = lck_grp_alloc_init(lck_name,
        cp->mc_sync_lock_grp = lck_grp_alloc_init(lck_name,

-           cp->mc_sync_lock_grp_attr);
-       cp->mc_sync_lock_attr = lck_attr_alloc_init();
-       lck_mtx_init(&cp->mc_sync_lock, cp->mc_sync_lock_grp,
-           cp->mc_sync_lock_attr);
+           LCK_GRP_ATTR_NULL);
+       lck_mtx_init(&cp->mc_sync_lock, cp->mc_sync_lock_grp, LCK_ATTR_NULL);

 
        for (btp = mcache_bkttype; chunksize <= btp->bt_minbuf; btp++) {
                continue;
 
        for (btp = mcache_bkttype; chunksize <= btp->bt_minbuf; btp++) {
                continue;


@@ -412,8 +379,7 @@ mcache_create_common(const char *name, size_t bufsize, size_t align,
                mcache_cpu_t *ccp = &cp->mc_cpu[c];
 
                VERIFY(IS_P2ALIGNED(ccp, CPU_CACHE_LINE_SIZE));
                mcache_cpu_t *ccp = &cp->mc_cpu[c];
 
                VERIFY(IS_P2ALIGNED(ccp, CPU_CACHE_LINE_SIZE));

-               lck_mtx_init(&ccp->cc_lock, cp->mc_cpu_lock_grp,
-                   cp->mc_cpu_lock_attr);
+               lck_mtx_init(&ccp->cc_lock, cp->mc_cpu_lock_grp, LCK_ATTR_NULL);

                ccp->cc_objs = -1;
                ccp->cc_pobjs = -1;
        }
                ccp->cc_objs = -1;
                ccp->cc_pobjs = -1;
        }


@@ -896,17 +862,9 @@ mcache_destroy(mcache_t *cp)
        cp->mc_slab_free = NULL;
        cp->mc_slab_audit = NULL;
 
        cp->mc_slab_free = NULL;
        cp->mc_slab_audit = NULL;
 

-       lck_attr_free(cp->mc_bkt_lock_attr);

        lck_grp_free(cp->mc_bkt_lock_grp);
        lck_grp_free(cp->mc_bkt_lock_grp);

-       lck_grp_attr_free(cp->mc_bkt_lock_grp_attr);
-
-       lck_attr_free(cp->mc_cpu_lock_attr);

        lck_grp_free(cp->mc_cpu_lock_grp);
        lck_grp_free(cp->mc_cpu_lock_grp);

-       lck_grp_attr_free(cp->mc_cpu_lock_grp_attr);
-
-       lck_attr_free(cp->mc_sync_lock_attr);

        lck_grp_free(cp->mc_sync_lock_grp);
        lck_grp_free(cp->mc_sync_lock_grp);

-       lck_grp_attr_free(cp->mc_sync_lock_grp_attr);

 
        /*
         * TODO: We need to destroy the zone here, but cannot do it
 
        /*
         * TODO: We need to destroy the zone here, but cannot do it


@@ -1358,7 +1316,7 @@ mcache_cache_update(mcache_t *cp)
        int need_bkt_resize = 0;
        int need_bkt_reenable = 0;
 
        int need_bkt_resize = 0;
        int need_bkt_reenable = 0;
 

-       lck_mtx_assert(mcache_llock, LCK_MTX_ASSERT_OWNED);
+       lck_mtx_assert(&mcache_llock, LCK_MTX_ASSERT_OWNED);

 
        mcache_bkt_ws_update(cp);
 
 
        mcache_bkt_ws_update(cp);
 


@@ -1645,13 +1603,9 @@ mcache_audit_free_verify_set(mcache_audit_t *mca, void *base, size_t offset,
 #define MCA_TRN_PREV ((mca->mca_next_trn + mca_trn_max - 1) % mca_trn_max)
 
 __private_extern__ char *
 #define MCA_TRN_PREV ((mca->mca_next_trn + mca_trn_max - 1) % mca_trn_max)
 
 __private_extern__ char *

-mcache_dump_mca(mcache_audit_t *mca)
+mcache_dump_mca(char buf[static DUMP_MCA_BUF_SIZE], mcache_audit_t *mca)

 {
 {

-       if (mca_dump_buf == NULL) {
-               return NULL;
-       }
-
-       snprintf(mca_dump_buf, DUMP_MCA_BUF_SIZE,
+       snprintf(buf, DUMP_MCA_BUF_SIZE,

            "mca %p: addr %p, cache %p (%s) nxttrn %d\n"
            DUMP_TRN_FMT()
            DUMP_TRN_FMT(),
            "mca %p: addr %p, cache %p (%s) nxttrn %d\n"
            DUMP_TRN_FMT()
            DUMP_TRN_FMT(),


@@ -1663,13 +1617,15 @@ mcache_dump_mca(mcache_audit_t *mca)
            DUMP_TRN_FIELDS("last", MCA_TRN_LAST),
            DUMP_TRN_FIELDS("previous", MCA_TRN_PREV));
 
            DUMP_TRN_FIELDS("last", MCA_TRN_LAST),
            DUMP_TRN_FIELDS("previous", MCA_TRN_PREV));
 

-       return mca_dump_buf;
+       return buf;

 }
 
 __private_extern__ void
 mcache_audit_panic(mcache_audit_t *mca, void *addr, size_t offset,
     int64_t expected, int64_t got)
 {
 }
 
 __private_extern__ void
 mcache_audit_panic(mcache_audit_t *mca, void *addr, size_t offset,
     int64_t expected, int64_t got)
 {

+       char buf[DUMP_MCA_BUF_SIZE];
+

        if (mca == NULL) {
                panic("mcache_audit: buffer %p modified after free at "
                    "offset 0x%lx (0x%llx instead of 0x%llx)\n", addr,
        if (mca == NULL) {
                panic("mcache_audit: buffer %p modified after free at "
                    "offset 0x%lx (0x%llx instead of 0x%llx)\n", addr,


@@ -1680,7 +1636,7 @@ mcache_audit_panic(mcache_audit_t *mca, void *addr, size_t offset,
 
        panic("mcache_audit: buffer %p modified after free at offset 0x%lx "
            "(0x%llx instead of 0x%llx)\n%s\n",
 
        panic("mcache_audit: buffer %p modified after free at offset 0x%lx "
            "(0x%llx instead of 0x%llx)\n%s\n",

-           addr, offset, got, expected, mcache_dump_mca(mca));
+           addr, offset, got, expected, mcache_dump_mca(buf, mca));

        /* NOTREACHED */
        __builtin_unreachable();
 }
        /* NOTREACHED */
        __builtin_unreachable();
 }