bsd/kern/kern_malloc.c

   1 /*
   2  * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
  29 /*
  30  * Copyright (c) 1987, 1991, 1993
  31  *      The Regents of the University of California.  All rights reserved.
  32  *
  33  * Redistribution and use in source and binary forms, with or without
  34  * modification, are permitted provided that the following conditions
  35  * are met:
  36  * 1. Redistributions of source code must retain the above copyright
  37  *    notice, this list of conditions and the following disclaimer.
  38  * 2. Redistributions in binary form must reproduce the above copyright
  39  *    notice, this list of conditions and the following disclaimer in the
  40  *    documentation and/or other materials provided with the distribution.
  41  * 3. All advertising materials mentioning features or use of this software
  42  *    must display the following acknowledgement:
  43  *      This product includes software developed by the University of
  44  *      California, Berkeley and its contributors.
  45  * 4. Neither the name of the University nor the names of its contributors
  46  *    may be used to endorse or promote products derived from this software
  47  *    without specific prior written permission.
  48  *
  49  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  50  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  51  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  52  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  53  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  54  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  55  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  56  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  57  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  58  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  59  * SUCH DAMAGE.
  60  *
  61  *      @(#)kern_malloc.c       8.4 (Berkeley) 5/20/95
  62  */
  63 /*
  64  * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
  65  * support for mandatory and extensible security protections.  This notice
  66  * is included in support of clause 2.2 (b) of the Apple Public License,
  67  * Version 2.0.
  68  */
  69
  70 #include <sys/param.h>
  71 #include <sys/malloc.h>
  72
  73 #include <sys/socket.h>
  74 #include <sys/socketvar.h>
  75
  76 #include <net/route.h>
  77 #include <net/necp.h>
  78
  79 #include <netinet/in.h>
  80 #include <netinet/in_systm.h>
  81 #include <netinet/ip.h>
  82 #include <netinet/in_pcb.h>
  83 #include <netinet/flow_divert.h>
  84
  85 #include <sys/event.h>
  86 #include <sys/eventvar.h>
  87
  88 #include <sys/proc_internal.h>
  89 #include <sys/mount_internal.h>
  90 #include <sys/vnode_internal.h>
  91 #include <sys/ubc_internal.h>
  92 #include <sys/namei.h>
  93 #include <sys/file_internal.h>
  94 #include <sys/filedesc.h>
  95 #include <sys/tty.h>
  96 #include <sys/quota.h>
  97 #include <sys/uio_internal.h>
  98 #include <sys/resourcevar.h>
  99 #include <sys/signalvar.h>
 100
 101 #include <hfs/hfs_cnode.h>
 102
 103 #include <miscfs/specfs/specdev.h>
 104
 105 #include <nfs/rpcv2.h>
 106 #include <nfs/nfsproto.h>
 107 #include <nfs/nfsnode.h>
 108 #include <nfs/nfsmount.h>
 109
 110 #include <vfs/vfs_journal.h>
 111
 112 #include <mach/mach_types.h>
 113
 114 #include <kern/zalloc.h>
 115 #include <kern/kalloc.h>
 116
 117 void kmeminit(void);
 118
 119 /* Strings corresponding to types of memory.
 120  * Must be in synch with the #defines is sys/malloc.h
 121  * NOTE - the reason we pass null strings in some cases is to reduce of foot
 122  * print as much as possible for systems where a tiny kernel is needed.
 123  * todo - We should probably redsign this and use enums for our types and only
 124  * include types needed for that configuration of the kernel.  This can't be
 125  * done without some kind of kpi since several types are hardwired and exported
 126  * (for example see types M_HFSMNT, M_UDFMNT, M_TEMP, etc in sys/malloc.h)
 127  */
 128 const char *memname[] = {
 129         "free",         /* 0 M_FREE */
 130         "mbuf",         /* 1 M_MBUF */
 131         "devbuf",       /* 2 M_DEVBUF */
 132         "socket",       /* 3 M_SOCKET */
 133         "pcb",          /* 4 M_PCB */
 134         "routetbl",     /* 5 M_RTABLE */
 135         "hosttbl",      /* 6 M_HTABLE */
 136         "fragtbl",      /* 7 M_FTABLE */
 137         "zombie",       /* 8 M_ZOMBIE */
 138         "ifaddr",       /* 9 M_IFADDR */
 139         "soopts",       /* 10 M_SOOPTS */
 140         "soname",       /* 11 M_SONAME */
 141         "namei",        /* 12 M_NAMEI */
 142         "gprof",        /* 13 M_GPROF */
 143         "ioctlops",     /* 14 M_IOCTLOPS */
 144         "mapmem",       /* 15 M_MAPMEM */
 145         "cred",         /* 16 M_CRED */
 146         "pgrp",         /* 17 M_PGRP */
 147         "session",      /* 18 M_SESSION */
 148         "iov32",        /* 19 M_IOV32 */
 149         "mount",        /* 20 M_MOUNT */
 150         "fhandle",              /* 21 M_FHANDLE */
 151 #if (NFSCLIENT || NFSSERVER)
 152         "NFS req",              /* 22 M_NFSREQ */
 153         "NFS mount",    /* 23 M_NFSMNT */
 154         "NFS node",             /* 24 M_NFSNODE */
 155 #else
 156         "",                             /* 22 M_NFSREQ */
 157         "",                             /* 23 M_NFSMNT */
 158         "",                             /* 24 M_NFSNODE */
 159 #endif
 160         "vnodes",               /* 25 M_VNODE */
 161         "namecache",    /* 26 M_CACHE */
 162 #if QUOTA
 163         "UFS quota",    /* 27 M_DQUOT */
 164 #else
 165         "",                             /* 27 M_DQUOT */
 166 #endif
 167         "proc uuid policy",             /* 28 M_PROC_UUID_POLICY */
 168 #if (SYSV_SEM || SYSV_MSG || SYSV_SHM)
 169         "shm",                  /* 29 M_SHM */
 170 #else
 171         "",                             /* 29 M_SHM */
 172 #endif
 173         "plimit",               /* 30 M_VMMAP */
 174         "sigacts",      /* 31 M_VMMAPENT */
 175         "VM object",    /* 32 M_VMOBJ */
 176         "VM objhash",   /* 33 M_VMOBJHASH */
 177         "VM pmap",              /* 34 M_VMPMAP */
 178         "VM pvmap",             /* 35 M_VMPVENT */
 179         "VM pager",             /* 36 M_VMPAGER */
 180         "VM pgdata",    /* 37 M_VMPGDATA */
 181         "fileproc",             /* 38 M_FILEPROC */
 182         "file desc",    /* 39 M_FILEDESC */
 183         "lockf",                /* 40 M_LOCKF */
 184         "proc",                 /* 41 M_PROC */
 185         "pstats",               /* 42 M_SUBPROC */
 186         "LFS segment",  /* 43 M_SEGMENT */
 187         "LFS node",             /* 44 M_LFSNODE */
 188         "",                             /* 45 M_FFSNODE */
 189         "MFS node",             /* 46 M_MFSNODE */
 190         "NQNFS Lease",  /* 47 M_NQLEASE */
 191         "NQNFS Host",   /* 48 M_NQMHOST */
 192         "Export Host",  /* 49 M_NETADDR */
 193 #if (NFSCLIENT || NFSSERVER)
 194         "NFS srvsock",  /* 50 M_NFSSVC */
 195         "NFS uid",              /* 51 M_NFSUID */
 196         "NFS daemon",   /* 52 M_NFSD */
 197 #else
 198         "",                             /* 50 M_NFSSVC */
 199         "",                             /* 51 M_NFSUID */
 200         "",                             /* 52 M_NFSD */
 201 #endif
 202         "ip_moptions",  /* 53 M_IPMOPTS */
 203         "in_multi",             /* 54 M_IPMADDR */
 204         "ether_multi",  /* 55 M_IFMADDR */
 205         "mrt",                  /* 56 M_MRTABLE */
 206         "",             /* 57 unused entry */
 207         "",             /* 58 unused entry */
 208 #if (NFSCLIENT || NFSSERVER)
 209         "NFSV3 srvdesc",/* 59 M_NFSRVDESC */
 210         "NFSV3 diroff", /* 60 M_NFSDIROFF */
 211         "NFSV3 bigfh",  /* 61 M_NFSBIGFH */
 212 #else
 213         "",                             /* 59 M_NFSRVDESC */
 214         "",                             /* 60 M_NFSDIROFF */
 215         "",                             /* 61 M_NFSBIGFH */
 216 #endif
 217         "MSDOSFS mount",/* 62 M_MSDOSFSMNT */
 218         "MSDOSFS fat",  /* 63 M_MSDOSFSFAT */
 219         "MSDOSFS node", /* 64 M_MSDOSFSNODE */
 220         "ttys",                 /* 65 M_TTYS */
 221         "exec",                 /* 66 M_EXEC */
 222         "miscfs mount", /* 67 M_MISCFSMNT */
 223         "miscfs node",  /* 68 M_MISCFSNODE */
 224         "adosfs mount", /* 69 M_ADOSFSMNT */
 225         "adosfs node",  /* 70 M_ADOSFSNODE */
 226         "adosfs anode", /* 71 M_ANODE */
 227         "buf hdrs",             /* 72 M_BUFHDR */
 228         "ofile tabl",   /* 73 M_OFILETABL */
 229         "mbuf clust",   /* 74 M_MCLUST */
 230 #if HFS
 231         "HFS mount",    /* 75 M_HFSMNT */
 232         "HFS node",             /* 76 M_HFSNODE */
 233         "HFS fork",             /* 77 M_HFSFORK */
 234 #else
 235         "",                             /* 75 M_HFSMNT */
 236         "",                             /* 76 M_HFSNODE */
 237         "",                             /* 77 M_HFSFORK */
 238 #endif
 239         "",     /* 78 unused */
 240         "",     /* 79 unused */
 241         "temp",                 /* 80 M_TEMP */
 242         "key mgmt",             /* 81 M_SECA */
 243         "DEVFS",                /* 82 M_DEVFS */
 244         "IpFw/IpAcct",  /* 83 M_IPFW */
 245         "UDF node",             /* 84 M_UDFNODE */
 246         "UDF mount",    /* 85 M_UDFMNT */
 247 #if INET6
 248         "IPv6 NDP",             /* 86 M_IP6NDP */
 249         "IPv6 options", /* 87 M_IP6OPT */
 250         "IPv6 Misc",    /* 88 M_IP6MISC */
 251 #else
 252         "",                             /* 86 M_IP6NDP */
 253         "",                             /* 87 M_IP6OPT */
 254         "",                             /* 88 M_IP6MISC */
 255 #endif
 256         "TCP Segment Q",/* 89 M_TSEGQ */
 257         "IGMP state",   /* 90 M_IGMP */
 258 #if JOURNALING
 259         "Journal",              /* 91 M_JNL_JNL */
 260         "Transaction",  /* 92 M_JNL_TR */
 261 #else
 262         "",                     /* 91 M_JNL_JNL */
 263         "",                     /* 92 M_JNL_TR */
 264 #endif
 265         "specinfo",             /* 93 M_SPECINFO */
 266         "kqueue",               /* 94 M_KQUEUE */
 267 #if HFS
 268         "HFS dirhint",  /* 95 M_HFSDIRHINT */
 269 #else
 270         "",                             /* 95 M_HFSDIRHINT */
 271 #endif
 272         "cluster_read", /* 96 M_CLRDAHEAD */
 273         "cluster_write",/* 97 M_CLWRBEHIND */
 274         "iov64",                /* 98 M_IOV64 */
 275         "fileglob",             /* 99 M_FILEGLOB */
 276         "kauth",                /* 100 M_KAUTH */
 277         "dummynet",             /* 101 M_DUMMYNET */
 278         "",                     /* 102 M_UNSAFEFS */
 279         "macpipelabel", /* 103 M_MACPIPELABEL */
 280         "mactemp",      /* 104 M_MACTEMP */
 281         "sbuf",         /* 105 M_SBUF */
 282         "extattr",      /* 106 M_EXTATTR */
 283         "select",       /* 107 M_SELECT */
 284 #if TRAFFIC_MGT
 285         "traffic_mgt",   /* 108 M_TRAFFIC_MGT */
 286 #else
 287         "", /* 108 M_TRAFFIC_MGT */
 288 #endif
 289 #if HFS_COMPRESSION
 290         "decmpfs_cnode",/* 109 M_DECMPFS_CNODE */
 291 #else
 292         "",             /* 109 M_DECMPFS_CNODE */
 293 #endif /* HFS_COMPRESSION */
 294         "ipmfilter",    /* 110 M_INMFILTER */
 295         "ipmsource",    /* 111 M_IPMSOURCE */
 296         "in6mfilter",   /* 112 M_IN6MFILTER */
 297         "ip6mopts",     /* 113 M_IP6MOPTS */
 298         "ip6msource",   /* 114 M_IP6MSOURCE */
 299 #if FLOW_DIVERT
 300         "flow_divert_pcb",      /* 115 M_FLOW_DIVERT_PCB */
 301         "flow_divert_group",    /* 116 M_FLOW_DIVERT_GROUP */
 302 #else
 303         "",                                     /* 115 M_FLOW_DIVERT_PCB */
 304         "",                                     /* 116 M_FLOW_DIVERT_GROUP */
 305 #endif
 306         "ip6cga",       /* 117 M_IP6CGA */
 307 #if NECP
 308         "necp",                                 /* 118 M_NECP */
 309         "necp_session_policy",  /* 119 M_NECP_SESSION_POLICY */
 310         "necp_socket_policy",   /* 120 M_NECP_SOCKET_POLICY */
 311         "necp_ip_policy",               /* 121 M_NECP_IP_POLICY */
 312 #else
 313         "",                                             /* 118 M_NECP */
 314         "",                                             /* 119 M_NECP_SESSION_POLICY */
 315         "",                                             /* 120 M_NECP_SOCKET_POLICY */
 316         "",                                             /* 121 M_NECP_IP_POLICY */
 317 #endif
 318         "fdvnodedata"   /* 122 M_FD_VN_DATA */
 319         "fddirbuf",     /* 123 M_FD_DIRBUF */
 320         "netagent",     /* 124 M_NETAGENT */
 321         ""
 322 };
 323
 324 /* for use with kmzones.kz_zalloczone */
 325 #define KMZ_CREATEZONE_ACCT     ((void *)-3)
 326 #define KMZ_CREATEZONE          ((void *)-2)
 327 #define KMZ_LOOKUPZONE          ((void *)-1)
 328 #define KMZ_MALLOC                      ((void *)0)
 329 #define KMZ_SHAREZONE           ((void *)1)
 330
 331 struct kmzones {
 332         size_t          kz_elemsize;
 333         void            *kz_zalloczone;
 334         boolean_t       kz_noencrypt;
 335 } kmzones[M_LAST] = {
 336 #define SOS(sname)      sizeof (struct sname)
 337 #define SOX(sname)      -1
 338         { -1,           0, FALSE },                     /* 0 M_FREE */
 339         { MSIZE,        KMZ_CREATEZONE, FALSE },        /* 1 M_MBUF */
 340         { 0,            KMZ_MALLOC, FALSE },            /* 2 M_DEVBUF */
 341         { SOS(socket),  KMZ_CREATEZONE, TRUE },         /* 3 M_SOCKET */
 342         { SOS(inpcb),   KMZ_LOOKUPZONE, TRUE },         /* 4 M_PCB */
 343         { M_MBUF,       KMZ_SHAREZONE, FALSE },         /* 5 M_RTABLE */
 344         { M_MBUF,       KMZ_SHAREZONE, FALSE },         /* 6 M_HTABLE */
 345         { M_MBUF,       KMZ_SHAREZONE, FALSE },         /* 7 M_FTABLE */
 346         { SOS(rusage),  KMZ_CREATEZONE, TRUE },         /* 8 M_ZOMBIE */
 347         { 0,            KMZ_MALLOC, FALSE },            /* 9 M_IFADDR */
 348         { M_MBUF,       KMZ_SHAREZONE, FALSE },         /* 10 M_SOOPTS */
 349         { 0,            KMZ_MALLOC, FALSE },            /* 11 M_SONAME */
 350         { MAXPATHLEN,   KMZ_CREATEZONE, FALSE },        /* 12 M_NAMEI */
 351         { 0,            KMZ_MALLOC, FALSE },            /* 13 M_GPROF */
 352         { 0,            KMZ_MALLOC, FALSE },            /* 14 M_IOCTLOPS */
 353         { 0,            KMZ_MALLOC, FALSE },            /* 15 M_MAPMEM */
 354         { SOS(ucred),   KMZ_CREATEZONE, FALSE },        /* 16 M_CRED */
 355         { SOS(pgrp),    KMZ_CREATEZONE, FALSE },        /* 17 M_PGRP */
 356         { SOS(session), KMZ_CREATEZONE, FALSE },        /* 18 M_SESSION */
 357         { SOS(user32_iovec),    KMZ_LOOKUPZONE, FALSE },/* 19 M_IOV32 */
 358         { SOS(mount),   KMZ_CREATEZONE, FALSE },        /* 20 M_MOUNT */
 359         { 0,            KMZ_MALLOC, FALSE },            /* 21 M_FHANDLE */
 360 #if (NFSCLIENT || NFSSERVER)
 361         { SOS(nfsreq),  KMZ_CREATEZONE, FALSE },        /* 22 M_NFSREQ */
 362         { SOS(nfsmount),KMZ_CREATEZONE, FALSE },        /* 23 M_NFSMNT */
 363         { SOS(nfsnode), KMZ_CREATEZONE, FALSE },        /* 24 M_NFSNODE */
 364 #else
 365         { 0,            KMZ_MALLOC, FALSE },            /* 22 M_NFSREQ */
 366         { 0,            KMZ_MALLOC, FALSE },            /* 23 M_NFSMNT */
 367         { 0,            KMZ_MALLOC, FALSE },            /* 24 M_NFSNODE */
 368 #endif
 369         { SOS(vnode),   KMZ_CREATEZONE, TRUE },         /* 25 M_VNODE */
 370         { SOS(namecache), KMZ_CREATEZONE, FALSE },      /* 26 M_CACHE */
 371 #if QUOTA
 372         { SOX(dquot),   KMZ_LOOKUPZONE, FALSE },        /* 27 M_DQUOT */
 373 #else
 374         { 0,            KMZ_MALLOC, FALSE },            /* 27 M_DQUOT */
 375 #endif
 376         { 0,            KMZ_MALLOC, FALSE },            /* 28 M_PROC_UUID_POLICY */
 377         { 0,            KMZ_MALLOC, FALSE },            /* 29 M_SHM */
 378         { SOS(plimit),  KMZ_CREATEZONE, TRUE },         /* 30 M_PLIMIT */
 379         { SOS(sigacts), KMZ_CREATEZONE_ACCT, TRUE },    /* 31 M_SIGACTS */
 380         { 0,            KMZ_MALLOC, FALSE },            /* 32 M_VMOBJ */
 381         { 0,            KMZ_MALLOC, FALSE },            /* 33 M_VMOBJHASH */
 382         { 0,            KMZ_MALLOC, FALSE },            /* 34 M_VMPMAP */
 383         { 0,            KMZ_MALLOC, FALSE },            /* 35 M_VMPVENT */
 384         { 0,            KMZ_MALLOC, FALSE },            /* 36 M_VMPAGER */
 385         { 0,            KMZ_MALLOC, FALSE },            /* 37 M_VMPGDATA */
 386         { SOS(fileproc),KMZ_CREATEZONE_ACCT, TRUE },    /* 38 M_FILEPROC */
 387         { SOS(filedesc),KMZ_CREATEZONE_ACCT, TRUE },    /* 39 M_FILEDESC */
 388         { SOX(lockf),   KMZ_CREATEZONE_ACCT, TRUE },    /* 40 M_LOCKF */
 389         { SOS(proc),    KMZ_CREATEZONE, FALSE },        /* 41 M_PROC */
 390         { SOS(pstats),  KMZ_CREATEZONE, TRUE },         /* 42 M_PSTATS */
 391         { 0,            KMZ_MALLOC, FALSE },            /* 43 M_SEGMENT */
 392         { M_FFSNODE,    KMZ_SHAREZONE, FALSE },         /* 44 M_LFSNODE */
 393         { 0,            KMZ_MALLOC, FALSE },            /* 45 M_FFSNODE */
 394         { M_FFSNODE,    KMZ_SHAREZONE, FALSE },         /* 46 M_MFSNODE */
 395         { 0,            KMZ_MALLOC, FALSE },            /* 47 M_NQLEASE */
 396         { 0,            KMZ_MALLOC, FALSE },            /* 48 M_NQMHOST */
 397         { 0,            KMZ_MALLOC, FALSE },            /* 49 M_NETADDR */
 398 #if (NFSCLIENT || NFSSERVER)
 399         { SOX(nfsrv_sock),
 400                         KMZ_CREATEZONE_ACCT, FALSE },   /* 50 M_NFSSVC */
 401         { 0,            KMZ_MALLOC, FALSE },            /* 51 M_NFSUID */
 402         { SOX(nfsrvcache),
 403                         KMZ_CREATEZONE_ACCT, FALSE },   /* 52 M_NFSD */
 404 #else
 405         { 0,            KMZ_MALLOC, FALSE },            /* 50 M_NFSSVC */
 406         { 0,            KMZ_MALLOC, FALSE },            /* 51 M_NFSUID */
 407         { 0,            KMZ_MALLOC, FALSE },            /* 52 M_NFSD */
 408 #endif
 409         { SOX(ip_moptions),
 410                         KMZ_LOOKUPZONE, FALSE },        /* 53 M_IPMOPTS */
 411         { SOX(in_multi),KMZ_LOOKUPZONE, FALSE },        /* 54 M_IPMADDR */
 412         { SOX(ether_multi),
 413                         KMZ_LOOKUPZONE, FALSE },        /* 55 M_IFMADDR */
 414         { SOX(mrt),     KMZ_CREATEZONE, TRUE },         /* 56 M_MRTABLE */
 415         { 0,            KMZ_MALLOC, FALSE },            /* 57 unused entry */
 416         { 0,            KMZ_MALLOC, FALSE },            /* 58 unused entry */
 417 #if (NFSCLIENT || NFSSERVER)
 418         { SOS(nfsrv_descript),
 419                         KMZ_CREATEZONE_ACCT, FALSE },   /* 59 M_NFSRVDESC */
 420         { SOS(nfsdmap), KMZ_CREATEZONE, FALSE },        /* 60 M_NFSDIROFF */
 421         { SOS(fhandle), KMZ_LOOKUPZONE, FALSE },        /* 61 M_NFSBIGFH */
 422 #else
 423         { 0,            KMZ_MALLOC, FALSE },            /* 59 M_NFSRVDESC */
 424         { 0,            KMZ_MALLOC, FALSE },            /* 60 M_NFSDIROFF */
 425         { 0,            KMZ_MALLOC, FALSE },            /* 61 M_NFSBIGFH */
 426 #endif
 427         { 0,            KMZ_MALLOC, FALSE },            /* 62 M_MSDOSFSMNT */
 428         { 0,            KMZ_MALLOC, FALSE },            /* 63 M_MSDOSFSFAT */
 429         { 0,            KMZ_MALLOC, FALSE },            /* 64 M_MSDOSFSNODE */
 430         { SOS(tty),     KMZ_CREATEZONE, FALSE },        /* 65 M_TTYS */
 431         { 0,            KMZ_MALLOC, FALSE },            /* 66 M_EXEC */
 432         { 0,            KMZ_MALLOC, FALSE },            /* 67 M_MISCFSMNT */
 433         { 0,            KMZ_MALLOC, FALSE },            /* 68 M_MISCFSNODE */
 434         { 0,            KMZ_MALLOC, FALSE },            /* 69 M_ADOSFSMNT */
 435         { 0,            KMZ_MALLOC, FALSE },            /* 70 M_ADOSFSNODE */
 436         { 0,            KMZ_MALLOC, FALSE },            /* 71 M_ANODE */
 437         { 0,            KMZ_MALLOC, TRUE },             /* 72 M_BUFHDR */
 438         { (NDFILE * OFILESIZE),
 439                         KMZ_CREATEZONE_ACCT, FALSE },   /* 73 M_OFILETABL */
 440         { MCLBYTES,     KMZ_CREATEZONE, FALSE },        /* 74 M_MCLUST */
 441 #if HFS
 442         { SOX(hfsmount),KMZ_LOOKUPZONE, FALSE },        /* 75 M_HFSMNT */
 443         { SOS(cnode),   KMZ_CREATEZONE, TRUE },         /* 76 M_HFSNODE */
 444         { SOS(filefork),KMZ_CREATEZONE, TRUE },         /* 77 M_HFSFORK */
 445 #else
 446         { 0,            KMZ_MALLOC, FALSE },            /* 75 M_HFSMNT */
 447         { 0,            KMZ_MALLOC, FALSE },            /* 76 M_HFSNODE */
 448         { 0,            KMZ_MALLOC, FALSE },            /* 77 M_HFSFORK */
 449 #endif
 450         { 0,            KMZ_MALLOC, FALSE },            /* 78 unused */
 451         { 0,            KMZ_MALLOC, FALSE },            /* 79 unused */
 452         { 0,            KMZ_MALLOC, FALSE },            /* 80 M_TEMP */
 453         { 0,            KMZ_MALLOC, FALSE },            /* 81 M_SECA */
 454         { 0,            KMZ_MALLOC, FALSE },            /* 82 M_DEVFS */
 455         { 0,            KMZ_MALLOC, FALSE },            /* 83 M_IPFW */
 456         { 0,            KMZ_MALLOC, FALSE },            /* 84 M_UDFNODE */
 457         { 0,            KMZ_MALLOC, FALSE },            /* 85 M_UDFMOUNT */
 458         { 0,            KMZ_MALLOC, FALSE },            /* 86 M_IP6NDP */
 459         { 0,            KMZ_MALLOC, FALSE },            /* 87 M_IP6OPT */
 460         { 0,            KMZ_MALLOC, FALSE },            /* 88 M_IP6MISC */
 461         { 0,            KMZ_MALLOC, FALSE },            /* 89 M_TSEGQ */
 462         { 0,            KMZ_MALLOC, FALSE },            /* 90 M_IGMP */
 463 #if JOURNALING
 464         { SOS(journal), KMZ_CREATEZONE, FALSE },        /* 91 M_JNL_JNL */
 465         { SOS(transaction), KMZ_CREATEZONE, FALSE },    /* 92 M_JNL_TR */
 466 #else
 467         { 0,            KMZ_MALLOC, FALSE },            /* 91 M_JNL_JNL */
 468         { 0,            KMZ_MALLOC, FALSE },            /* 92 M_JNL_TR */
 469 #endif
 470         { SOS(specinfo),KMZ_CREATEZONE, TRUE },         /* 93 M_SPECINFO */
 471         { SOS(kqueue),  KMZ_CREATEZONE, FALSE },        /* 94 M_KQUEUE */
 472 #if HFS
 473         { SOS(directoryhint), KMZ_CREATEZONE, TRUE },   /* 95 M_HFSDIRHINT */
 474 #else
 475         { 0,            KMZ_MALLOC, FALSE },            /* 95 M_HFSDIRHINT */
 476 #endif
 477         { SOS(cl_readahead),  KMZ_CREATEZONE, TRUE },   /* 96 M_CLRDAHEAD */
 478         { SOS(cl_writebehind),KMZ_CREATEZONE, TRUE },   /* 97 M_CLWRBEHIND */
 479         { SOS(user64_iovec),    KMZ_LOOKUPZONE, FALSE },/* 98 M_IOV64 */
 480         { SOS(fileglob),        KMZ_CREATEZONE, TRUE }, /* 99 M_FILEGLOB */
 481         { 0,            KMZ_MALLOC, FALSE },            /* 100 M_KAUTH */
 482         { 0,            KMZ_MALLOC, FALSE },            /* 101 M_DUMMYNET */
 483         { 0,            KMZ_MALLOC, FALSE },            /* 102 M_UNSAFEFS */
 484         { 0,            KMZ_MALLOC, FALSE },            /* 103 M_MACPIPELABEL */
 485         { 0,            KMZ_MALLOC, FALSE },            /* 104 M_MACTEMP */
 486         { 0,            KMZ_MALLOC, FALSE },            /* 105 M_SBUF */
 487         { 0,            KMZ_MALLOC, FALSE },            /* 106 M_HFS_EXTATTR */
 488         { 0,            KMZ_MALLOC, FALSE },            /* 107 M_SELECT */
 489         { 0,            KMZ_MALLOC, FALSE },            /* 108 M_TRAFFIC_MGT */
 490 #if HFS_COMPRESSION
 491         { SOS(decmpfs_cnode),KMZ_CREATEZONE , FALSE},   /* 109 M_DECMPFS_CNODE */
 492 #else
 493         { 0,            KMZ_MALLOC, FALSE },            /* 109 M_DECMPFS_CNODE */
 494 #endif /* HFS_COMPRESSION */
 495         { 0,            KMZ_MALLOC, FALSE },            /* 110 M_INMFILTER */
 496         { 0,            KMZ_MALLOC, FALSE },            /* 111 M_IPMSOURCE */
 497         { 0,            KMZ_MALLOC, FALSE },            /* 112 M_IN6MFILTER */
 498         { 0,            KMZ_MALLOC, FALSE },            /* 113 M_IP6MOPTS */
 499         { 0,            KMZ_MALLOC, FALSE },            /* 114 M_IP6MSOURCE */
 500 #if FLOW_DIVERT
 501         { SOS(flow_divert_pcb),         KMZ_CREATEZONE, TRUE }, /* 115 M_FLOW_DIVERT_PCB */
 502         { SOS(flow_divert_group),       KMZ_CREATEZONE, TRUE }, /* 116 M_FLOW_DIVERT_GROUP */
 503 #else
 504         { 0,            KMZ_MALLOC, FALSE },            /* 115 M_FLOW_DIVERT_PCB */
 505         { 0,            KMZ_MALLOC, FALSE },            /* 116 M_FLOW_DIVERT_GROUP */
 506 #endif  /* FLOW_DIVERT */
 507         { 0,            KMZ_MALLOC, FALSE },            /* 117 M_IP6CGA */
 508         { 0,            KMZ_MALLOC, FALSE },            /* 118 M_NECP */
 509 #if NECP
 510         { SOS(necp_session_policy),     KMZ_CREATEZONE, TRUE }, /* 119 M_NECP_SESSION_POLICY */
 511         { SOS(necp_kernel_socket_policy),       KMZ_CREATEZONE, TRUE }, /* 120 M_NECP_SOCKET_POLICY */
 512         { SOS(necp_kernel_ip_output_policy),    KMZ_CREATEZONE, TRUE }, /* 121 M_NECP_IP_POLICY */
 513 #else
 514         { 0,            KMZ_MALLOC, FALSE },            /* 119 M_NECP_SESSION_POLICY */
 515         { 0,            KMZ_MALLOC, FALSE },            /* 120 M_NECP_SOCKET_POLICY */
 516         { 0,            KMZ_MALLOC, FALSE },            /* 121 M_NECP_IP_POLICY */
 517 #endif /* NECP */
 518         { 0,            KMZ_MALLOC, FALSE },            /* 122 M_FD_VN_DATA */
 519         { 0,            KMZ_MALLOC, FALSE },            /* 123 M_FD_DIRBUF */
 520         { 0,            KMZ_MALLOC, FALSE },            /* 124 M_NETAGENT */
 521 #undef  SOS
 522 #undef  SOX
 523 };
 524
 525 extern zone_t kalloc_zone(vm_size_t);   /* XXX */
 526
 527 /*
 528  * Initialize the kernel memory allocator
 529  */
 530 void
 531 kmeminit(void)
 532 {
 533         struct kmzones  *kmz;
 534
 535         if ((sizeof(kmzones)/sizeof(kmzones[0])) != (sizeof(memname)/sizeof(memname[0]))) {
 536                 panic("kmeminit: kmzones has %lu elements but memname has %lu\n",
 537                           (sizeof(kmzones)/sizeof(kmzones[0])), (sizeof(memname)/sizeof(memname[0])));
 538         }
 539
 540         kmz = kmzones;
 541         while (kmz < &kmzones[M_LAST]) {
 542 /* XXX */
 543                 if (kmz->kz_elemsize == (size_t)(-1))
 544                         ;
 545                 else
 546 /* XXX */
 547                 if (kmz->kz_zalloczone == KMZ_CREATEZONE ||
 548                     kmz->kz_zalloczone == KMZ_CREATEZONE_ACCT) {
 549                         kmz->kz_zalloczone = zinit(kmz->kz_elemsize,
 550                                                 1024 * 1024, PAGE_SIZE,
 551                                                 memname[kmz - kmzones]);
 552                         zone_change(kmz->kz_zalloczone, Z_CALLERACCT,
 553                                     (kmz->kz_zalloczone == KMZ_CREATEZONE_ACCT));
 554
 555                         if (kmz->kz_noencrypt == TRUE)
 556                                 zone_change(kmz->kz_zalloczone, Z_NOENCRYPT, TRUE);
 557                 }
 558                 else if (kmz->kz_zalloczone == KMZ_LOOKUPZONE)
 559                         kmz->kz_zalloczone = kalloc_zone(kmz->kz_elemsize);
 560
 561                 kmz++;
 562         }
 563
 564         kmz = kmzones;
 565         while (kmz < &kmzones[M_LAST]) {
 566 /* XXX */
 567                 if (kmz->kz_elemsize == (size_t)(-1))
 568                         ;
 569                 else
 570 /* XXX */
 571                 if (kmz->kz_zalloczone == KMZ_SHAREZONE) {
 572                         kmz->kz_zalloczone =
 573                                 kmzones[kmz->kz_elemsize].kz_zalloczone;
 574                         kmz->kz_elemsize =
 575                                 kmzones[kmz->kz_elemsize].kz_elemsize;
 576                 }
 577
 578                 kmz++;
 579         }
 580 }
 581
 582 struct _mhead {
 583         size_t  mlen;
 584         char    dat[0];
 585 };
 586
 587
 588 void *
 589 _MALLOC_external(
 590         size_t          size,
 591         int             type,
 592         int             flags);
 593 void *
 594 _MALLOC_external(
 595         size_t          size,
 596         int             type,
 597         int             flags)
 598 {
 599     static vm_allocation_site_t site = { VM_KERN_MEMORY_KALLOC, VM_TAG_BT };
 600     return (__MALLOC(size, type, flags, &site));
 601 }
 602
 603 void *
 604 __MALLOC(
 605         size_t          size,
 606         int             type,
 607         int             flags,
 608         vm_allocation_site_t *site)
 609 {
 610         struct _mhead   *hdr = NULL;
 611         size_t          memsize = sizeof (*hdr) + size;
 612
 613         if (type >= M_LAST)
 614                 panic("_malloc TYPE");
 615
 616         if (size == 0)
 617                 return (NULL);
 618
 619         if (flags & M_NOWAIT) {
 620                if (size > memsize)   /* overflow detected */
 621                        return (NULL);
 622                else
 623                        hdr = (void *)kalloc_canblock(memsize, FALSE, site);
 624         } else {
 625                if (size > memsize) {
 626                        /*
 627                         * We get here when the caller told us to block, waiting for memory but an overflow
 628                         * has been detected.  The caller isn't expecting a NULL return code so we panic
 629                         * with a descriptive message.
 630                         */
 631                        panic("_MALLOC: overflow detected, size %llu ", (uint64_t) size);
 632                }
 633                else
 634                        hdr = (void *)kalloc_canblock(memsize, TRUE, site);
 635
 636                if (hdr == NULL) {
 637
 638                         /*
 639                          * We get here when the caller told us to block waiting for memory, but
 640                          * kalloc said there's no memory left to get.  Generally, this means there's a
 641                          * leak or the caller asked for an impossibly large amount of memory.  Since there's
 642                          * nothing left to wait for and the caller isn't expecting a NULL return code, we
 643                          * just panic.  This is less than ideal, but returning NULL doesn't help since the
 644                          * majority of callers don't check the return value and will just dereference the pointer and
 645                          * trap anyway.  We may as well get a more descriptive message out while we can.
 646                          */
 647
 648                         panic("_MALLOC: kalloc returned NULL (potential leak), size %llu", (uint64_t) size);
 649                 }
 650         }
 651         if (!hdr)
 652                 return (0);
 653
 654         hdr->mlen = memsize;
 655
 656         if (flags & M_ZERO)
 657                 bzero(hdr->dat, size);
 658
 659         return  (hdr->dat);
 660 }
 661
 662 void
 663 _FREE(
 664         void            *addr,
 665         int             type)
 666 {
 667         struct _mhead   *hdr;
 668
 669         if (type >= M_LAST)
 670                 panic("_free TYPE");
 671
 672         if (!addr)
 673                 return; /* correct (convenient bsd kernel legacy) */
 674
 675         hdr = addr; hdr--;
 676         kfree(hdr, hdr->mlen);
 677 }
 678
 679 void *
 680 __REALLOC(
 681         void            *addr,
 682         size_t          size,
 683         int             type,
 684         int             flags,
 685         vm_allocation_site_t *site)
 686 {
 687         struct _mhead   *hdr;
 688         void            *newaddr;
 689         size_t          alloc;
 690
 691         /* realloc(NULL, ...) is equivalent to malloc(...) */
 692         if (addr == NULL)
 693                 return (__MALLOC(size, type, flags, site));
 694
 695         /* Allocate a new, bigger (or smaller) block */
 696         if ((newaddr = __MALLOC(size, type, flags, site)) == NULL)
 697                 return (NULL);
 698
 699         hdr = addr;
 700         --hdr;
 701         alloc = hdr->mlen - sizeof (*hdr);
 702
 703         /* Copy over original contents */
 704         bcopy(addr, newaddr, MIN(size, alloc));
 705         _FREE(addr, type);
 706
 707         return (newaddr);
 708 }
 709
 710 void *
 711 _MALLOC_ZONE_external(
 712         size_t          size,
 713         int             type,
 714         int             flags);
 715 void *
 716 _MALLOC_ZONE_external(
 717         size_t          size,
 718         int             type,
 719         int             flags)
 720 {
 721     return (__MALLOC_ZONE(size, type, flags, NULL));
 722 }
 723
 724 void *
 725 __MALLOC_ZONE(
 726         size_t          size,
 727         int             type,
 728         int             flags,
 729         vm_allocation_site_t *site)
 730 {
 731         struct kmzones  *kmz;
 732         void            *elem;
 733
 734         if (type >= M_LAST)
 735                 panic("_malloc_zone TYPE");
 736
 737         kmz = &kmzones[type];
 738         if (kmz->kz_zalloczone == KMZ_MALLOC)
 739                 panic("_malloc_zone ZONE: type = %d", type);
 740
 741 /* XXX */
 742         if (kmz->kz_elemsize == (size_t)(-1))
 743                 panic("_malloc_zone XXX");
 744 /* XXX */
 745         if (size == kmz->kz_elemsize)
 746                 if (flags & M_NOWAIT) {
 747                         elem = (void *)zalloc_noblock(kmz->kz_zalloczone);
 748                 } else {
 749                         elem = (void *)zalloc(kmz->kz_zalloczone);
 750                 }
 751         else
 752                 if (flags & M_NOWAIT) {
 753                         elem = (void *)kalloc_canblock(size, FALSE, site);
 754                 } else {
 755                         elem = (void *)kalloc_canblock(size, TRUE, site);
 756                 }
 757
 758         return (elem);
 759 }
 760
 761 void
 762 _FREE_ZONE(
 763         void            *elem,
 764         size_t          size,
 765         int             type)
 766 {
 767         struct kmzones  *kmz;
 768
 769         if (type >= M_LAST)
 770                 panic("FREE_SIZE");
 771
 772         kmz = &kmzones[type];
 773         if (kmz->kz_zalloczone == KMZ_MALLOC)
 774                 panic("free_zone ZONE");
 775
 776 /* XXX */
 777         if (kmz->kz_elemsize == (size_t)(-1))
 778                 panic("FREE_SIZE XXX");
 779 /* XXX */
 780         if (size == kmz->kz_elemsize)
 781                 zfree(kmz->kz_zalloczone, elem);
 782         else
 783                 kfree(elem, size);
 784 }
 785
 786 #if CONFIG_ZLEAKS
 787
 788 SYSCTL_DECL(_kern_zleak);
 789 SYSCTL_NODE(_kern, OID_AUTO, zleak, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "zleak");
 790
 791 /*
 792  * kern.zleak.active
 793  *
 794  * Show the status of the zleak subsystem (0 = enabled, 1 = active,
 795  * and -1 = failed), and if enabled, allow it to be activated immediately.
 796  */
 797 static int
 798 sysctl_zleak_active SYSCTL_HANDLER_ARGS
 799 {
 800 #pragma unused(arg1, arg2)
 801         int oldval, val, error;
 802
 803         val = oldval = get_zleak_state();
 804         error = sysctl_handle_int(oidp, &val, 0, req);
 805         if (error || !req->newptr)
 806                 return (error);
 807         /*
 808          * Can only be activated if it's off (and not failed.)
 809          * Cannot be deactivated once it's on.
 810          */
 811         if (val == 1 && oldval == 0) {
 812                 kern_return_t kr = zleak_activate();
 813
 814                 if (KERN_SUCCESS != kr)
 815                         printf("zleak_active: failed to activate "
 816                             "live zone leak debugging (%d).\n", kr);
 817         } if (val == 0 && oldval == 1) {
 818                 printf("zleak_active: active, cannot be disabled.\n");
 819                 return (EINVAL);
 820         }
 821         return (0);
 822 }
 823
 824 SYSCTL_PROC(_kern_zleak, OID_AUTO, active,
 825     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
 826     0, 0, sysctl_zleak_active, "I", "zleak activity");
 827
 828 /*
 829  * kern.zleak.max_zonemap_size
 830  *
 831  * Read the value of the maximum zonemap size in bytes; useful
 832  * as the maximum size that zleak.global_threshold and
 833  * zleak.zone_threshold should be set to.
 834  */
 835 static int
 836 sysctl_zleak_max_zonemap_size SYSCTL_HANDLER_ARGS
 837 {
 838         uint64_t zmap_max_size = *(vm_size_t *)arg1;
 839
 840         return sysctl_handle_quad(oidp, &zmap_max_size, arg2, req);
 841 }
 842
 843 SYSCTL_PROC(_kern_zleak, OID_AUTO, max_zonemap_size,
 844     CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED,
 845     &zleak_max_zonemap_size, 0,
 846     sysctl_zleak_max_zonemap_size, "Q", "zleak max zonemap size");
 847
 848
 849 static int
 850 sysctl_zleak_threshold SYSCTL_HANDLER_ARGS
 851 {
 852 #pragma unused(oidp, arg2)
 853         int error;
 854         uint64_t value = *(vm_size_t *)arg1;
 855
 856         error = sysctl_io_number(req, value, sizeof (value), &value, NULL);
 857
 858         if (error || !req->newptr)
 859                 return (error);
 860
 861         if (value > (uint64_t)zleak_max_zonemap_size)
 862                 return (ERANGE);
 863
 864         *(vm_size_t *)arg1 = value;
 865         return (0);
 866 }
 867
 868 /*
 869  * kern.zleak.global_threshold
 870  *
 871  * Set the global zleak threshold size (in bytes).  If the zone map
 872  * grows larger than this value, zleaks are automatically activated.
 873  *
 874  * The default value is set in zleak_init().
 875  */
 876 SYSCTL_PROC(_kern_zleak, OID_AUTO, global_threshold,
 877     CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
 878     &zleak_global_tracking_threshold, 0,
 879     sysctl_zleak_threshold, "Q", "zleak global threshold");
 880
 881 /*
 882  * kern.zleak.zone_threshold
 883  *
 884  * Set the per-zone threshold size (in bytes) above which any
 885  * zone will automatically start zleak tracking.
 886  *
 887  * The default value is set in zleak_init().
 888  *
 889  * Setting this variable will have no effect until zleak tracking is
 890  * activated (See above.)
 891  */
 892 SYSCTL_PROC(_kern_zleak, OID_AUTO, zone_threshold,
 893     CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED,
 894     &zleak_per_zone_tracking_threshold, 0,
 895     sysctl_zleak_threshold, "Q", "zleak per-zone threshold");
 896
 897 #endif  /* CONFIG_ZLEAKS */