bsd/kern/kern_malloc.c

   1 /*
   2  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
  29 /*
  30  * Copyright (c) 1987, 1991, 1993
  31  *      The Regents of the University of California.  All rights reserved.
  32  *
  33  * Redistribution and use in source and binary forms, with or without
  34  * modification, are permitted provided that the following conditions
  35  * are met:
  36  * 1. Redistributions of source code must retain the above copyright
  37  *    notice, this list of conditions and the following disclaimer.
  38  * 2. Redistributions in binary form must reproduce the above copyright
  39  *    notice, this list of conditions and the following disclaimer in the
  40  *    documentation and/or other materials provided with the distribution.
  41  * 3. All advertising materials mentioning features or use of this software
  42  *    must display the following acknowledgement:
  43  *      This product includes software developed by the University of
  44  *      California, Berkeley and its contributors.
  45  * 4. Neither the name of the University nor the names of its contributors
  46  *    may be used to endorse or promote products derived from this software
  47  *    without specific prior written permission.
  48  *
  49  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  50  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  51  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  52  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  53  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  54  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  55  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  56  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  57  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  58  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  59  * SUCH DAMAGE.
  60  *
  61  *      @(#)kern_malloc.c       8.4 (Berkeley) 5/20/95
  62  */
  63 /*
  64  * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
  65  * support for mandatory and extensible security protections.  This notice
  66  * is included in support of clause 2.2 (b) of the Apple Public License,
  67  * Version 2.0.
  68  */
  69
  70 #include <sys/param.h>
  71 #include <sys/malloc.h>
  72
  73 #include <sys/socket.h>
  74 #include <sys/socketvar.h>
  75
  76 #include <net/route.h>
  77
  78 #include <netinet/in.h>
  79 #include <netinet/in_systm.h>
  80 #include <netinet/ip.h>
  81 #include <netinet/in_pcb.h>
  82
  83 #include <sys/event.h>
  84 #include <sys/eventvar.h>
  85
  86 #include <sys/proc_internal.h>
  87 #include <sys/mount_internal.h>
  88 #include <sys/vnode_internal.h>
  89 #include <sys/ubc_internal.h>
  90 #include <sys/namei.h>
  91 #include <sys/file_internal.h>
  92 #include <sys/filedesc.h>
  93 #include <sys/tty.h>
  94 #include <sys/quota.h>
  95 #include <sys/uio_internal.h>
  96 #include <sys/resourcevar.h>
  97 #include <sys/signalvar.h>
  98
  99 #include <hfs/hfs_cnode.h>
 100
 101 #include <miscfs/specfs/specdev.h>
 102
 103 #include <nfs/rpcv2.h>
 104 #include <nfs/nfsproto.h>
 105 #include <nfs/nfsnode.h>
 106 #include <nfs/nfsmount.h>
 107
 108 #include <vfs/vfs_journal.h>
 109
 110 #include <mach/mach_types.h>
 111
 112 #include <kern/zalloc.h>
 113 #include <kern/kalloc.h>
 114
 115 void kmeminit(void) __attribute__((section("__TEXT, initcode")));
 116
 117 /* Strings corresponding to types of memory.
 118  * Must be in synch with the #defines is sys/malloc.h
 119  * NOTE - the reason we pass null strings in some cases is to reduce of foot
 120  * print as much as possible for systems where a tiny kernel is needed.
 121  * todo - We should probably redsign this and use enums for our types and only
 122  * include types needed for that configuration of the kernel.  This can't be
 123  * done without some kind of kpi since several types are hardwired and exported
 124  * (for example see types M_HFSMNT, M_UDFMNT, M_TEMP, etc in sys/malloc.h)
 125  */
 126 const char *memname[] = {
 127         "free",         /* 0 M_FREE */
 128         "mbuf",         /* 1 M_MBUF */
 129         "devbuf",       /* 2 M_DEVBUF */
 130         "socket",       /* 3 M_SOCKET */
 131         "pcb",          /* 4 M_PCB */
 132         "routetbl",     /* 5 M_RTABLE */
 133         "hosttbl",      /* 6 M_HTABLE */
 134         "fragtbl",      /* 7 M_FTABLE */
 135         "zombie",       /* 8 M_ZOMBIE */
 136         "ifaddr",       /* 9 M_IFADDR */
 137         "soopts",       /* 10 M_SOOPTS */
 138         "soname",       /* 11 M_SONAME */
 139         "namei",        /* 12 M_NAMEI */
 140         "gprof",        /* 13 M_GPROF */
 141         "ioctlops",     /* 14 M_IOCTLOPS */
 142         "mapmem",       /* 15 M_MAPMEM */
 143         "cred",         /* 16 M_CRED */
 144         "pgrp",         /* 17 M_PGRP */
 145         "session",      /* 18 M_SESSION */
 146         "iov32",        /* 19 M_IOV32 */
 147         "mount",        /* 20 M_MOUNT */
 148         "fhandle",              /* 21 M_FHANDLE */
 149 #if (NFSCLIENT || NFSSERVER)
 150         "NFS req",              /* 22 M_NFSREQ */
 151         "NFS mount",    /* 23 M_NFSMNT */
 152         "NFS node",             /* 24 M_NFSNODE */
 153 #else
 154         "",                             /* 22 M_NFSREQ */
 155         "",                             /* 23 M_NFSMNT */
 156         "",                             /* 24 M_NFSNODE */
 157 #endif
 158         "vnodes",               /* 25 M_VNODE */
 159         "namecache",    /* 26 M_CACHE */
 160 #if QUOTA
 161         "UFS quota",    /* 27 M_DQUOT */
 162 #else
 163         "",                             /* 27 M_DQUOT */
 164 #endif
 165         "",                             /* 28 M_UFSMNT */
 166 #if (SYSV_SEM || SYSV_MSG || SYSV_SHM)
 167         "shm",                  /* 29 M_SHM */
 168 #else
 169         "",                             /* 29 M_SHM */
 170 #endif
 171         "plimit",               /* 30 M_VMMAP */
 172         "sigacts",      /* 31 M_VMMAPENT */
 173         "VM object",    /* 32 M_VMOBJ */
 174         "VM objhash",   /* 33 M_VMOBJHASH */
 175         "VM pmap",              /* 34 M_VMPMAP */
 176         "VM pvmap",             /* 35 M_VMPVENT */
 177         "VM pager",             /* 36 M_VMPAGER */
 178         "VM pgdata",    /* 37 M_VMPGDATA */
 179         "fileproc",             /* 38 M_FILEPROC */
 180         "file desc",    /* 39 M_FILEDESC */
 181         "lockf",                /* 40 M_LOCKF */
 182         "proc",                 /* 41 M_PROC */
 183         "pstats",               /* 42 M_SUBPROC */
 184         "LFS segment",  /* 43 M_SEGMENT */
 185         "LFS node",             /* 44 M_LFSNODE */
 186         "",                             /* 45 M_FFSNODE */
 187         "MFS node",             /* 46 M_MFSNODE */
 188         "NQNFS Lease",  /* 47 M_NQLEASE */
 189         "NQNFS Host",   /* 48 M_NQMHOST */
 190         "Export Host",  /* 49 M_NETADDR */
 191 #if (NFSCLIENT || NFSSERVER)
 192         "NFS srvsock",  /* 50 M_NFSSVC */
 193         "NFS uid",              /* 51 M_NFSUID */
 194         "NFS daemon",   /* 52 M_NFSD */
 195 #else
 196         "",                             /* 50 M_NFSSVC */
 197         "",                             /* 51 M_NFSUID */
 198         "",                             /* 52 M_NFSD */
 199 #endif
 200         "ip_moptions",  /* 53 M_IPMOPTS */
 201         "in_multi",             /* 54 M_IPMADDR */
 202         "ether_multi",  /* 55 M_IFMADDR */
 203         "mrt",                  /* 56 M_MRTABLE */
 204         "",             /* 57 unused entry */
 205         "",             /* 58 unused entry */
 206 #if (NFSCLIENT || NFSSERVER)
 207         "NFSV3 srvdesc",/* 59 M_NFSRVDESC */
 208         "NFSV3 diroff", /* 60 M_NFSDIROFF */
 209         "NFSV3 bigfh",  /* 61 M_NFSBIGFH */
 210 #else
 211         "",                             /* 59 M_NFSRVDESC */
 212         "",                             /* 60 M_NFSDIROFF */
 213         "",                             /* 61 M_NFSBIGFH */
 214 #endif
 215         "MSDOSFS mount",/* 62 M_MSDOSFSMNT */
 216         "MSDOSFS fat",  /* 63 M_MSDOSFSFAT */
 217         "MSDOSFS node", /* 64 M_MSDOSFSNODE */
 218         "ttys",                 /* 65 M_TTYS */
 219         "exec",                 /* 66 M_EXEC */
 220         "miscfs mount", /* 67 M_MISCFSMNT */
 221         "miscfs node",  /* 68 M_MISCFSNODE */
 222         "adosfs mount", /* 69 M_ADOSFSMNT */
 223         "adosfs node",  /* 70 M_ADOSFSNODE */
 224         "adosfs anode", /* 71 M_ANODE */
 225         "buf hdrs",             /* 72 M_BUFHDR */
 226         "ofile tabl",   /* 73 M_OFILETABL */
 227         "mbuf clust",   /* 74 M_MCLUST */
 228 #if HFS
 229         "HFS mount",    /* 75 M_HFSMNT */
 230         "HFS node",             /* 76 M_HFSNODE */
 231         "HFS fork",             /* 77 M_HFSFORK */
 232 #else
 233         "",                             /* 75 M_HFSMNT */
 234         "",                             /* 76 M_HFSNODE */
 235         "",                             /* 77 M_HFSFORK */
 236 #endif
 237         "ZFS mount",    /* 78 M_ZFSFSMNT */
 238         "ZFS node",     /* 79 M_ZFSNODE */
 239         "temp",                 /* 80 M_TEMP */
 240         "key mgmt",             /* 81 M_SECA */
 241         "DEVFS",                /* 82 M_DEVFS */
 242         "IpFw/IpAcct",  /* 83 M_IPFW */
 243         "UDF node",             /* 84 M_UDFNODE */
 244         "UDF mount",    /* 85 M_UDFMNT */
 245 #if INET6
 246         "IPv6 NDP",             /* 86 M_IP6NDP */
 247         "IPv6 options", /* 87 M_IP6OPT */
 248         "IPv6 Misc",    /* 88 M_IP6MISC */
 249 #else
 250         "",                             /* 86 M_IP6NDP */
 251         "",                             /* 87 M_IP6OPT */
 252         "",                             /* 88 M_IP6MISC */
 253 #endif
 254         "TCP Segment Q",/* 89 M_TSEGQ */
 255         "IGMP state",   /* 90 M_IGMP */
 256 #if JOURNALING
 257         "Journal",              /* 91 M_JNL_JNL */
 258         "Transaction",  /* 92 M_JNL_TR */
 259 #else
 260         "",                     /* 91 M_JNL_JNL */
 261         "",                     /* 92 M_JNL_TR */
 262 #endif
 263         "specinfo",             /* 93 M_SPECINFO */
 264         "kqueue",               /* 94 M_KQUEUE */
 265 #if HFS
 266         "HFS dirhint",  /* 95 M_HFSDIRHINT */
 267 #else
 268         "",                             /* 95 M_HFSDIRHINT */
 269 #endif
 270         "cluster_read", /* 96 M_CLRDAHEAD */
 271         "cluster_write",/* 97 M_CLWRBEHIND */
 272         "iov64",                /* 98 M_IOV64 */
 273         "fileglob",             /* 99 M_FILEGLOB */
 274         "kauth",                /* 100 M_KAUTH */
 275         "dummynet",             /* 101 M_DUMMYNET */
 276 #ifndef __LP64__
 277         "unsafe_fsnode",        /* 102 M_UNSAFEFS */
 278 #else
 279         "",                     /* 102 M_UNSAFEFS */
 280 #endif /* __LP64__ */
 281         "macpipelabel", /* 103 M_MACPIPELABEL */
 282         "mactemp",      /* 104 M_MACTEMP */
 283         "sbuf",         /* 105 M_SBUF */
 284         "extattr",      /* 106 M_EXTATTR */
 285         "lctx",         /* 107 M_LCTX */
 286 #if TRAFFIC_MGT
 287         "traffic_mgt",   /* 108 M_TRAFFIC_MGT */
 288 #else
 289         "", /* 108 M_TRAFFIC_MGT */
 290 #endif
 291 #if HFS_COMPRESSION
 292         "decmpfs_cnode",/* 109 M_DECMPFS_CNODE */
 293 #else
 294         "",             /* 109 M_DECMPFS_CNODE */
 295 #endif /* HFS_COMPRESSION */
 296 };
 297
 298 /* for use with kmzones.kz_zalloczone */
 299 #define KMZ_CREATEZONE          ((void *)-2)
 300 #define KMZ_LOOKUPZONE          ((void *)-1)
 301 #define KMZ_MALLOC                      ((void *)0)
 302 #define KMZ_SHAREZONE           ((void *)1)
 303
 304 struct kmzones {
 305         size_t          kz_elemsize;
 306         void            *kz_zalloczone;
 307 } kmzones[M_LAST] = {
 308 #define SOS(sname)      sizeof (struct sname)
 309 #define SOX(sname)      -1
 310         { -1,           0 },                    /* 0 M_FREE */
 311         { MSIZE,        KMZ_CREATEZONE },       /* 1 M_MBUF */
 312         { 0,            KMZ_MALLOC },           /* 2 M_DEVBUF */
 313         { SOS(socket),  KMZ_CREATEZONE },       /* 3 M_SOCKET */
 314         { SOS(inpcb),   KMZ_LOOKUPZONE },       /* 4 M_PCB */
 315         { M_MBUF,       KMZ_SHAREZONE },        /* 5 M_RTABLE */
 316         { M_MBUF,       KMZ_SHAREZONE },        /* 6 M_HTABLE */
 317         { M_MBUF,       KMZ_SHAREZONE },        /* 7 M_FTABLE */
 318         { SOS(rusage),  KMZ_CREATEZONE },       /* 8 M_ZOMBIE */
 319         { 0,            KMZ_MALLOC },           /* 9 M_IFADDR */
 320         { M_MBUF,       KMZ_SHAREZONE },                /* 10 M_SOOPTS */
 321         { 0,            KMZ_MALLOC },           /* 11 M_SONAME */
 322         { MAXPATHLEN,   KMZ_CREATEZONE },               /* 12 M_NAMEI */
 323         { 0,            KMZ_MALLOC },           /* 13 M_GPROF */
 324         { 0,            KMZ_MALLOC },           /* 14 M_IOCTLOPS */
 325         { 0,            KMZ_MALLOC },           /* 15 M_MAPMEM */
 326         { SOS(ucred),   KMZ_CREATEZONE },       /* 16 M_CRED */
 327         { SOS(pgrp),    KMZ_CREATEZONE },       /* 17 M_PGRP */
 328         { SOS(session), KMZ_CREATEZONE },       /* 18 M_SESSION */
 329         { SOS(user32_iovec),    KMZ_LOOKUPZONE },       /* 19 M_IOV32 */
 330         { SOS(mount),   KMZ_CREATEZONE },       /* 20 M_MOUNT */
 331         { 0,            KMZ_MALLOC },           /* 21 M_FHANDLE */
 332 #if (NFSCLIENT || NFSSERVER)
 333         { SOS(nfsreq),  KMZ_CREATEZONE },       /* 22 M_NFSREQ */
 334         { SOS(nfsmount),        KMZ_CREATEZONE },       /* 23 M_NFSMNT */
 335         { SOS(nfsnode), KMZ_CREATEZONE },       /* 24 M_NFSNODE */
 336 #else
 337         { 0,            KMZ_MALLOC },           /* 22 M_NFSREQ */
 338         { 0,            KMZ_MALLOC },           /* 23 M_NFSMNT */
 339         { 0,            KMZ_MALLOC },           /* 24 M_NFSNODE */
 340 #endif
 341         { SOS(vnode),   KMZ_CREATEZONE },       /* 25 M_VNODE */
 342         { SOS(namecache),       KMZ_CREATEZONE },       /* 26 M_CACHE */
 343 #if QUOTA
 344         { SOX(dquot),   KMZ_LOOKUPZONE },       /* 27 M_DQUOT */
 345 #else
 346         { 0,            KMZ_MALLOC },           /* 27 M_DQUOT */
 347 #endif
 348         { 0,            KMZ_MALLOC },           /* 28 M_UFSMNT */
 349         { 0,            KMZ_MALLOC },           /* 29 M_CGSUM */
 350         { SOS(plimit),  KMZ_CREATEZONE },       /* 30 M_PLIMIT */
 351         { SOS(sigacts), KMZ_CREATEZONE },       /* 31 M_SIGACTS */
 352         { 0,            KMZ_MALLOC },           /* 32 M_VMOBJ */
 353         { 0,            KMZ_MALLOC },           /* 33 M_VMOBJHASH */
 354         { 0,            KMZ_MALLOC },           /* 34 M_VMPMAP */
 355         { 0,            KMZ_MALLOC },           /* 35 M_VMPVENT */
 356         { 0,            KMZ_MALLOC },           /* 36 M_VMPAGER */
 357         { 0,            KMZ_MALLOC },           /* 37 M_VMPGDATA */
 358         { SOS(fileproc),        KMZ_CREATEZONE },       /* 38 M_FILEPROC */
 359         { SOS(filedesc),        KMZ_CREATEZONE },       /* 39 M_FILEDESC */
 360         { SOX(lockf),   KMZ_CREATEZONE },       /* 40 M_LOCKF */
 361         { SOS(proc),    KMZ_CREATEZONE },       /* 41 M_PROC */
 362         { SOS(pstats),  KMZ_CREATEZONE },       /* 42 M_PSTATS */
 363         { 0,            KMZ_MALLOC },           /* 43 M_SEGMENT */
 364         { M_FFSNODE,    KMZ_SHAREZONE },        /* 44 M_LFSNODE */
 365         { 0,            KMZ_MALLOC },           /* 45 M_FFSNODE */
 366         { M_FFSNODE,    KMZ_SHAREZONE },        /* 46 M_MFSNODE */
 367         { 0,            KMZ_MALLOC },           /* 47 M_NQLEASE */
 368         { 0,            KMZ_MALLOC },           /* 48 M_NQMHOST */
 369         { 0,            KMZ_MALLOC },           /* 49 M_NETADDR */
 370 #if (NFSCLIENT || NFSSERVER)
 371         { SOX(nfsrv_sock),
 372                         KMZ_CREATEZONE },       /* 50 M_NFSSVC */
 373         { 0,            KMZ_MALLOC },           /* 51 M_NFSUID */
 374         { SOX(nfsrvcache),
 375                         KMZ_CREATEZONE },       /* 52 M_NFSD */
 376 #else
 377         { 0,            KMZ_MALLOC },           /* 50 M_NFSSVC */
 378         { 0,            KMZ_MALLOC },           /* 51 M_NFSUID */
 379         { 0,            KMZ_MALLOC },           /* 52 M_NFSD */
 380 #endif
 381         { SOX(ip_moptions),
 382                         KMZ_LOOKUPZONE },       /* 53 M_IPMOPTS */
 383         { SOX(in_multi),        KMZ_LOOKUPZONE },       /* 54 M_IPMADDR */
 384         { SOX(ether_multi),
 385                         KMZ_LOOKUPZONE },       /* 55 M_IFMADDR */
 386         { SOX(mrt),     KMZ_CREATEZONE },       /* 56 M_MRTABLE */
 387         { 0,            KMZ_MALLOC },           /* 57 unused entry */
 388         { 0,            KMZ_MALLOC },           /* 58 unused entry */
 389 #if (NFSCLIENT || NFSSERVER)
 390         { SOS(nfsrv_descript),
 391                         KMZ_CREATEZONE },       /* 59 M_NFSRVDESC */
 392         { SOS(nfsdmap), KMZ_CREATEZONE },       /* 60 M_NFSDIROFF */
 393         { SOS(fhandle), KMZ_LOOKUPZONE },       /* 61 M_NFSBIGFH */
 394 #else
 395         { 0,            KMZ_MALLOC },           /* 59 M_NFSRVDESC */
 396         { 0,            KMZ_MALLOC },           /* 60 M_NFSDIROFF */
 397         { 0,            KMZ_MALLOC },           /* 61 M_NFSBIGFH */
 398 #endif
 399         { 0,            KMZ_MALLOC },           /* 62 M_MSDOSFSMNT */
 400         { 0,            KMZ_MALLOC },           /* 63 M_MSDOSFSFAT */
 401         { 0,            KMZ_MALLOC },           /* 64 M_MSDOSFSNODE */
 402         { SOS(tty),     KMZ_CREATEZONE },       /* 65 M_TTYS */
 403         { 0,            KMZ_MALLOC },           /* 66 M_EXEC */
 404         { 0,            KMZ_MALLOC },           /* 67 M_MISCFSMNT */
 405         { 0,            KMZ_MALLOC },           /* 68 M_MISCFSNODE */
 406         { 0,            KMZ_MALLOC },           /* 69 M_ADOSFSMNT */
 407         { 0,            KMZ_MALLOC },           /* 70 M_ADOSFSNODE */
 408         { 0,            KMZ_MALLOC },           /* 71 M_ANODE */
 409         { SOX(buf),     KMZ_CREATEZONE },       /* 72 M_BUFHDR */
 410         { (NDFILE * OFILESIZE),
 411                         KMZ_CREATEZONE },       /* 73 M_OFILETABL */
 412         { MCLBYTES,     KMZ_CREATEZONE },       /* 74 M_MCLUST */
 413 #if HFS
 414         { SOX(hfsmount),        KMZ_LOOKUPZONE },       /* 75 M_HFSMNT */
 415         { SOS(cnode),   KMZ_CREATEZONE },       /* 76 M_HFSNODE */
 416         { SOS(filefork),        KMZ_CREATEZONE },       /* 77 M_HFSFORK */
 417 #else
 418         { 0,            KMZ_MALLOC },           /* 75 M_HFSMNT */
 419         { 0,            KMZ_MALLOC },           /* 76 M_HFSNODE */
 420         { 0,            KMZ_MALLOC },           /* 77 M_HFSFORK */
 421 #endif
 422         { 0,            KMZ_MALLOC },           /* 78 M_ZFSMNT */
 423         { 0,            KMZ_MALLOC },           /* 79 M_ZFSNODE */
 424         { 0,            KMZ_MALLOC },           /* 80 M_TEMP */
 425         { 0,            KMZ_MALLOC },           /* 81 M_SECA */
 426         { 0,            KMZ_MALLOC },           /* 82 M_DEVFS */
 427         { 0,            KMZ_MALLOC },           /* 83 M_IPFW */
 428         { 0,            KMZ_MALLOC },           /* 84 M_UDFNODE */
 429         { 0,            KMZ_MALLOC },           /* 85 M_UDFMOUNT */
 430         { 0,            KMZ_MALLOC },           /* 86 M_IP6NDP */
 431         { 0,            KMZ_MALLOC },           /* 87 M_IP6OPT */
 432         { 0,            KMZ_MALLOC },           /* 88 M_IP6MISC */
 433         { 0,            KMZ_MALLOC },           /* 89 M_TSEGQ */
 434         { 0,            KMZ_MALLOC },           /* 90 M_IGMP */
 435 #if JOURNALING
 436         { SOS(journal), KMZ_CREATEZONE },       /* 91 M_JNL_JNL */
 437         { SOS(transaction), KMZ_CREATEZONE },   /* 92 M_JNL_TR */
 438 #else
 439         { 0,     KMZ_MALLOC },          /* 91 M_JNL_JNL */
 440         { 0,     KMZ_MALLOC },          /* 92 M_JNL_TR */
 441 #endif
 442         { SOS(specinfo), KMZ_CREATEZONE },      /* 93 M_SPECINFO */
 443         { SOS(kqueue), KMZ_CREATEZONE },        /* 94 M_KQUEUE */
 444 #if HFS
 445         { SOS(directoryhint), KMZ_CREATEZONE }, /* 95 M_HFSDIRHINT */
 446 #else
 447         { 0,    KMZ_MALLOC },           /* 95 M_HFSDIRHINT */
 448 #endif
 449         { SOS(cl_readahead),  KMZ_CREATEZONE }, /* 96 M_CLRDAHEAD */
 450         { SOS(cl_writebehind),KMZ_CREATEZONE }, /* 97 M_CLWRBEHIND */
 451         { SOS(user64_iovec),    KMZ_LOOKUPZONE },       /* 98 M_IOV64 */
 452         { SOS(fileglob),        KMZ_CREATEZONE },       /* 99 M_FILEGLOB */
 453         { 0,            KMZ_MALLOC },           /* 100 M_KAUTH */
 454         { 0,            KMZ_MALLOC },           /* 101 M_DUMMYNET */
 455 #ifndef __LP64__
 456         { SOS(unsafe_fsnode),KMZ_CREATEZONE },  /* 102 M_UNSAFEFS */
 457 #else
 458         { 0,            KMZ_MALLOC },           /* 102 M_UNSAFEFS */
 459 #endif /* __LP64__ */
 460         { 0,            KMZ_MALLOC },           /* 103 M_MACPIPELABEL */
 461         { 0,            KMZ_MALLOC },           /* 104 M_MACTEMP */
 462         { 0,            KMZ_MALLOC },           /* 105 M_SBUF */
 463         { 0,            KMZ_MALLOC },           /* 106 M_HFS_EXTATTR */
 464         { 0,            KMZ_MALLOC },           /* 107 M_LCTX */
 465         { 0,            KMZ_MALLOC },           /* 108 M_TRAFFIC_MGT */
 466 #if HFS_COMPRESSION
 467         { SOS(decmpfs_cnode),KMZ_CREATEZONE },  /* 109 M_DECMPFS_CNODE */
 468 #else
 469         { 0,            KMZ_MALLOC },           /* 109 M_DECMPFS_CNODE */
 470 #endif /* HFS_COMPRESSION */
 471 #undef  SOS
 472 #undef  SOX
 473 };
 474
 475 extern zone_t kalloc_zone(vm_size_t);   /* XXX */
 476
 477 /*
 478  * Initialize the kernel memory allocator
 479  */
 480 void
 481 kmeminit(void)
 482 {
 483         struct kmzones  *kmz;
 484
 485         if ((sizeof(kmzones)/sizeof(kmzones[0])) != (sizeof(memname)/sizeof(memname[0]))) {
 486                 panic("kmeminit: kmzones has %lu elements but memname has %lu\n",
 487                           (sizeof(kmzones)/sizeof(kmzones[0])), (sizeof(memname)/sizeof(memname[0])));
 488         }
 489
 490         kmz = kmzones;
 491         while (kmz < &kmzones[M_LAST]) {
 492 /* XXX */
 493                 if (kmz->kz_elemsize == (size_t)(-1))
 494                         ;
 495                 else
 496 /* XXX */
 497                 if (kmz->kz_zalloczone == KMZ_CREATEZONE) {
 498                         kmz->kz_zalloczone = zinit(kmz->kz_elemsize,
 499                                                 1024 * 1024, PAGE_SIZE,
 500                                                 memname[kmz - kmzones]);
 501                 }
 502                 else if (kmz->kz_zalloczone == KMZ_LOOKUPZONE)
 503                         kmz->kz_zalloczone = kalloc_zone(kmz->kz_elemsize);
 504
 505                 kmz++;
 506         }
 507
 508         kmz = kmzones;
 509         while (kmz < &kmzones[M_LAST]) {
 510 /* XXX */
 511                 if (kmz->kz_elemsize == (size_t)(-1))
 512                         ;
 513                 else
 514 /* XXX */
 515                 if (kmz->kz_zalloczone == KMZ_SHAREZONE) {
 516                         kmz->kz_zalloczone =
 517                                 kmzones[kmz->kz_elemsize].kz_zalloczone;
 518                         kmz->kz_elemsize =
 519                                 kmzones[kmz->kz_elemsize].kz_elemsize;
 520                 }
 521
 522                 kmz++;
 523         }
 524 }
 525
 526 #define MDECL(reqlen)                                   \
 527 union {                                                 \
 528         struct  _mhead hdr;                             \
 529         char    _m[(reqlen) + sizeof (struct _mhead)];  \
 530 }
 531
 532 struct _mhead {
 533         size_t  mlen;
 534         char    dat[0];
 535 };
 536
 537 void *
 538 _MALLOC(
 539         size_t          size,
 540         int             type,
 541         int             flags)
 542 {
 543         MDECL(size)     *mem;
 544         size_t          memsize = sizeof (*mem);
 545
 546         if (type >= M_LAST)
 547                 panic("_malloc TYPE");
 548
 549         if (size == 0)
 550                 return (NULL);
 551
 552         if (flags & M_NOWAIT) {
 553                 mem = (void *)kalloc_noblock(memsize);
 554         } else {
 555                 mem = (void *)kalloc(memsize);
 556
 557                 if (mem == NULL) {
 558
 559                         /*
 560                          * We get here when the caller told us to block waiting for memory, but
 561                          * kalloc said there's no memory left to get.  Generally, this means there's a
 562                          * leak or the caller asked for an impossibly large amount of memory.  Since there's
 563                          * nothing left to wait for and the caller isn't expecting a NULL return code, we
 564                          * just panic.  This is less than ideal, but returning NULL doesn't help since the
 565                          * majority of callers don't check the return value and will just dereference the pointer and
 566                          * trap anyway.  We may as well get a more descriptive message out while we can.
 567                          */
 568
 569                         panic("_MALLOC: kalloc returned NULL (potential leak), size %llu", (uint64_t) size);
 570                 }
 571         }
 572         if (!mem)
 573                 return (0);
 574
 575         mem->hdr.mlen = memsize;
 576
 577         if (flags & M_ZERO)
 578                 bzero(mem->hdr.dat, size);
 579
 580         return  (mem->hdr.dat);
 581 }
 582
 583 void
 584 _FREE(
 585         void            *addr,
 586         int             type)
 587 {
 588         struct _mhead   *hdr;
 589
 590         if (type >= M_LAST)
 591                 panic("_free TYPE");
 592
 593         if (!addr)
 594                 return; /* correct (convenient bsd kernel legacy) */
 595
 596         hdr = addr; hdr--;
 597         kfree(hdr, hdr->mlen);
 598 }
 599
 600 void *
 601 _MALLOC_ZONE(
 602         size_t          size,
 603         int             type,
 604         int             flags)
 605 {
 606         struct kmzones  *kmz;
 607         void            *elem;
 608
 609         if (type >= M_LAST)
 610                 panic("_malloc_zone TYPE");
 611
 612         kmz = &kmzones[type];
 613         if (kmz->kz_zalloczone == KMZ_MALLOC)
 614                 panic("_malloc_zone ZONE: type = %d", type);
 615
 616 /* XXX */
 617         if (kmz->kz_elemsize == (size_t)(-1))
 618                 panic("_malloc_zone XXX");
 619 /* XXX */
 620         if (size == kmz->kz_elemsize)
 621                 if (flags & M_NOWAIT) {
 622                         elem = (void *)zalloc_noblock(kmz->kz_zalloczone);
 623                 } else {
 624                         elem = (void *)zalloc(kmz->kz_zalloczone);
 625                 }
 626         else
 627                 if (flags & M_NOWAIT) {
 628                         elem = (void *)kalloc_noblock(size);
 629                 } else {
 630                         elem = (void *)kalloc(size);
 631                 }
 632
 633         return (elem);
 634 }
 635
 636 void
 637 _FREE_ZONE(
 638         void            *elem,
 639         size_t          size,
 640         int             type)
 641 {
 642         struct kmzones  *kmz;
 643
 644         if (type >= M_LAST)
 645                 panic("FREE_SIZE");
 646
 647         kmz = &kmzones[type];
 648         if (kmz->kz_zalloczone == KMZ_MALLOC)
 649                 panic("free_zone ZONE");
 650
 651 /* XXX */
 652         if (kmz->kz_elemsize == (size_t)(-1))
 653                 panic("FREE_SIZE XXX");
 654 /* XXX */
 655         if (size == kmz->kz_elemsize)
 656                 zfree(kmz->kz_zalloczone, elem);
 657         else
 658                 kfree(elem, size);
 659 }