bsd/kern/kern_malloc.c

   1 /*
   2  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
  31 /*
  32  * Copyright (c) 1987, 1991, 1993
  33  *      The Regents of the University of California.  All rights reserved.
  34  *
  35  * Redistribution and use in source and binary forms, with or without
  36  * modification, are permitted provided that the following conditions
  37  * are met:
  38  * 1. Redistributions of source code must retain the above copyright
  39  *    notice, this list of conditions and the following disclaimer.
  40  * 2. Redistributions in binary form must reproduce the above copyright
  41  *    notice, this list of conditions and the following disclaimer in the
  42  *    documentation and/or other materials provided with the distribution.
  43  * 3. All advertising materials mentioning features or use of this software
  44  *    must display the following acknowledgement:
  45  *      This product includes software developed by the University of
  46  *      California, Berkeley and its contributors.
  47  * 4. Neither the name of the University nor the names of its contributors
  48  *    may be used to endorse or promote products derived from this software
  49  *    without specific prior written permission.
  50  *
  51  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  52  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  53  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  54  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  55  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  56  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  57  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  58  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  59  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  60  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  61  * SUCH DAMAGE.
  62  *
  63  *      @(#)kern_malloc.c       8.4 (Berkeley) 5/20/95
  64  */
  65
  66 #include <sys/param.h>
  67 #include <sys/malloc.h>
  68
  69 #include <sys/socket.h>
  70 #include <sys/socketvar.h>
  71
  72 #include <net/route.h>
  73
  74 #include <netinet/in.h>
  75 #include <netinet/in_systm.h>
  76 #include <netinet/ip.h>
  77 #include <netinet/in_pcb.h>
  78
  79 #include <sys/event.h>
  80 #include <sys/eventvar.h>
  81
  82 #include <sys/proc_internal.h>
  83 #include <sys/mount_internal.h>
  84 #include <sys/vnode_internal.h>
  85 #include <sys/ubc_internal.h>
  86 #include <sys/namei.h>
  87 #include <sys/file_internal.h>
  88 #include <sys/filedesc.h>
  89 #include <sys/tty.h>
  90 #include <sys/quota.h>
  91 #include <sys/uio_internal.h>
  92 #include <sys/resourcevar.h>
  93
  94 #include <ufs/ufs/inode.h>
  95
  96 #include <hfs/hfs_cnode.h>
  97 #include <isofs/cd9660/cd9660_node.h>
  98
  99 #include <miscfs/volfs/volfs.h>
 100 #include <miscfs/specfs/specdev.h>
 101
 102 #include <nfs/rpcv2.h>
 103 #include <nfs/nfsproto.h>
 104 #include <nfs/nfsnode.h>
 105 #include <nfs/nfsmount.h>
 106
 107 #include <vfs/vfs_journal.h>
 108
 109 #include <mach/mach_types.h>
 110
 111 #include <kern/zalloc.h>
 112 #include <kern/kalloc.h>
 113
 114 struct kmemstats kmemstats[M_LAST];
 115 char *memname[] = INITKMEMNAMES;
 116
 117 struct kmzones {
 118         size_t          kz_elemsize;
 119         void            *kz_zalloczone;
 120 #define KMZ_CREATEZONE          ((void *)-2)
 121 #define KMZ_LOOKUPZONE          ((void *)-1)
 122 #define KMZ_MALLOC              ((void *)0)
 123 #define KMZ_SHAREZONE           ((void *)1)
 124 } kmzones[M_LAST] = {
 125 #define SOS(sname)      sizeof (struct sname)
 126 #define SOX(sname)      -1
 127         -1,             0,                      /* 0 M_FREE */
 128         MSIZE,          KMZ_CREATEZONE,         /* 1 M_MBUF */
 129         0,              KMZ_MALLOC,             /* 2 M_DEVBUF */
 130         SOS(socket),    KMZ_CREATEZONE,         /* 3 M_SOCKET */
 131         SOS(inpcb),     KMZ_LOOKUPZONE,         /* 4 M_PCB */
 132         M_MBUF,         KMZ_SHAREZONE,          /* 5 M_RTABLE */
 133         M_MBUF,         KMZ_SHAREZONE,          /* 6 M_HTABLE */
 134         M_MBUF,         KMZ_SHAREZONE,          /* 7 M_FTABLE */
 135         SOS(rusage),    KMZ_CREATEZONE,         /* 8 M_ZOMBIE */
 136         0,              KMZ_MALLOC,             /* 9 M_IFADDR */
 137         M_MBUF,         KMZ_SHAREZONE,          /* 10 M_SOOPTS */
 138         0,              KMZ_MALLOC,             /* 11 M_SONAME */
 139         MAXPATHLEN,     KMZ_CREATEZONE,         /* 12 M_NAMEI */
 140         0,              KMZ_MALLOC,             /* 13 M_GPROF */
 141         0,              KMZ_MALLOC,             /* 14 M_IOCTLOPS */
 142         0,              KMZ_MALLOC,             /* 15 M_MAPMEM */
 143         SOS(ucred),     KMZ_CREATEZONE,         /* 16 M_CRED */
 144         SOS(pgrp),      KMZ_CREATEZONE,         /* 17 M_PGRP */
 145         SOS(session),   KMZ_CREATEZONE,         /* 18 M_SESSION */
 146         SOS(iovec_32),  KMZ_LOOKUPZONE,         /* 19 M_IOV32 */
 147         SOS(mount),     KMZ_CREATEZONE,         /* 20 M_MOUNT */
 148         0,              KMZ_MALLOC,             /* 21 M_FHANDLE */
 149         SOS(nfsreq),    KMZ_CREATEZONE,         /* 22 M_NFSREQ */
 150         SOS(nfsmount),  KMZ_CREATEZONE,         /* 23 M_NFSMNT */
 151         SOS(nfsnode),   KMZ_CREATEZONE,         /* 24 M_NFSNODE */
 152         SOS(vnode),     KMZ_CREATEZONE,         /* 25 M_VNODE */
 153         SOS(namecache), KMZ_CREATEZONE,         /* 26 M_CACHE */
 154         SOX(dquot),     KMZ_LOOKUPZONE,         /* 27 M_DQUOT */
 155         SOX(ufsmount),  KMZ_LOOKUPZONE,         /* 28 M_UFSMNT */
 156         0,              KMZ_MALLOC,             /* 29 M_CGSUM */
 157         0,              KMZ_MALLOC,             /* 30 M_VMMAP */
 158         0,              KMZ_MALLOC,             /* 31 M_VMMAPENT */
 159         0,              KMZ_MALLOC,             /* 32 M_VMOBJ */
 160         0,              KMZ_MALLOC,             /* 33 M_VMOBJHASH */
 161         0,              KMZ_MALLOC,             /* 34 M_VMPMAP */
 162         0,              KMZ_MALLOC,             /* 35 M_VMPVENT */
 163         0,              KMZ_MALLOC,             /* 36 M_VMPAGER */
 164         0,              KMZ_MALLOC,             /* 37 M_VMPGDATA */
 165         SOS(fileproc),  KMZ_CREATEZONE,         /* 38 M_FILEPROC */
 166         SOS(filedesc),  KMZ_CREATEZONE,         /* 39 M_FILEDESC */
 167         SOX(lockf),     KMZ_CREATEZONE,         /* 40 M_LOCKF */
 168         SOS(proc),      KMZ_CREATEZONE,         /* 41 M_PROC */
 169         SOS(pstats),    KMZ_CREATEZONE,         /* 42 M_SUBPROC */
 170         0,              KMZ_MALLOC,             /* 43 M_SEGMENT */
 171         M_FFSNODE,      KMZ_SHAREZONE,          /* 44 M_LFSNODE */
 172         SOS(inode),     KMZ_CREATEZONE,         /* 45 M_FFSNODE */
 173         M_FFSNODE,      KMZ_SHAREZONE,          /* 46 M_MFSNODE */
 174         0,              KMZ_MALLOC,             /* 47 M_NQLEASE */
 175         0,              KMZ_MALLOC,             /* 48 M_NQMHOST */
 176         0,              KMZ_MALLOC,             /* 49 M_NETADDR */
 177         SOX(nfssvc_sock),
 178                         KMZ_CREATEZONE,         /* 50 M_NFSSVC */
 179         SOS(nfsuid),    KMZ_CREATEZONE,         /* 51 M_NFSUID */
 180         SOX(nfsrvcache),
 181                         KMZ_CREATEZONE,         /* 52 M_NFSD */
 182         SOX(ip_moptions),
 183                         KMZ_LOOKUPZONE,         /* 53 M_IPMOPTS */
 184         SOX(in_multi),  KMZ_LOOKUPZONE,         /* 54 M_IPMADDR */
 185         SOX(ether_multi),
 186                         KMZ_LOOKUPZONE,         /* 55 M_IFMADDR */
 187         SOX(mrt),       KMZ_CREATEZONE,         /* 56 M_MRTABLE */
 188         SOX(iso_mnt),   KMZ_LOOKUPZONE,         /* 57 M_ISOFSMNT */
 189         SOS(iso_node),  KMZ_CREATEZONE,         /* 58 M_ISOFSNODE */
 190         SOS(nfsrv_descript),
 191                         KMZ_CREATEZONE,         /* 59 M_NFSRVDESC */
 192         SOS(nfsdmap),   KMZ_CREATEZONE,         /* 60 M_NFSDIROFF */
 193         SOS(fhandle),   KMZ_LOOKUPZONE,         /* 61 M_NFSBIGFH */
 194         0,              KMZ_MALLOC,             /* 62 M_MSDOSFSMNT */
 195         0,              KMZ_MALLOC,             /* 63 M_MSDOSFSFAT */
 196         0,              KMZ_MALLOC,             /* 64 M_MSDOSFSNODE */
 197         SOS(tty),       KMZ_CREATEZONE,         /* 65 M_TTYS */
 198         0,              KMZ_MALLOC,             /* 66 M_EXEC */
 199         0,              KMZ_MALLOC,             /* 67 M_MISCFSMNT */
 200         0,              KMZ_MALLOC,             /* 68 M_MISCFSNODE */
 201         0,              KMZ_MALLOC,             /* 69 M_ADOSFSMNT */
 202         0,              KMZ_MALLOC,             /* 70 M_ADOSFSNODE */
 203         0,              KMZ_MALLOC,             /* 71 M_ANODE */
 204         SOX(buf),       KMZ_CREATEZONE,         /* 72 M_BUFHDR */
 205         (NDFILE * OFILESIZE),
 206                         KMZ_CREATEZONE,         /* 73 M_OFILETABL */
 207         MCLBYTES,       KMZ_CREATEZONE,         /* 74 M_MCLUST */
 208         SOX(hfsmount),  KMZ_LOOKUPZONE,         /* 75 M_HFSMNT */
 209         SOS(cnode),     KMZ_CREATEZONE,         /* 76 M_HFSNODE */
 210         SOS(filefork),  KMZ_CREATEZONE,         /* 77 M_HFSFORK */
 211         SOX(volfs_mntdata),     KMZ_LOOKUPZONE,         /* 78 M_VOLFSMNT */
 212         SOS(volfs_vndata),      KMZ_CREATEZONE,         /* 79 M_VOLFSNODE */
 213         0,              KMZ_MALLOC,             /* 80 M_TEMP */
 214         0,              KMZ_MALLOC,             /* 81 M_SECA */
 215         0,              KMZ_MALLOC,             /* 82 M_DEVFS */
 216         0,              KMZ_MALLOC,             /* 83 M_IPFW */
 217         0,              KMZ_MALLOC,             /* 84 M_UDFNODE */
 218         0,              KMZ_MALLOC,             /* 85 M_UDFMOUNT */
 219         0,              KMZ_MALLOC,             /* 86 M_IP6NDP */
 220         0,              KMZ_MALLOC,             /* 87 M_IP6OPT */
 221         0,              KMZ_MALLOC,             /* 88 M_IP6MISC */
 222         0,              KMZ_MALLOC,             /* 89 M_TSEGQ */
 223         0,              KMZ_MALLOC,             /* 90 M_IGMP */
 224         SOS(journal), KMZ_CREATEZONE,     /* 91 M_JNL_JNL */
 225         SOS(transaction), KMZ_CREATEZONE,     /* 92 M_JNL_TR */
 226         SOS(specinfo), KMZ_CREATEZONE,          /* 93 M_SPECINFO */
 227         SOS(kqueue), KMZ_CREATEZONE,            /* 94 M_KQUEUE */
 228         SOS(directoryhint), KMZ_CREATEZONE,     /* 95 M_HFSDIRHINT */
 229         SOS(cl_readahead),  KMZ_CREATEZONE,     /* 96 M_CLRDAHEAD */
 230         SOS(cl_writebehind),KMZ_CREATEZONE,     /* 97 M_CLWRBEHIND */
 231         SOS(iovec_64),  KMZ_LOOKUPZONE,         /* 98 M_IOV64 */
 232         SOS(fileglob),  KMZ_CREATEZONE,         /* 99 M_FILEGLOB */
 233         0,              KMZ_MALLOC,             /* 100 M_KAUTH */
 234         0,              KMZ_MALLOC,             /* 101 M_DUMMYNET */
 235         SOS(unsafe_fsnode),KMZ_CREATEZONE,      /* 102 M_UNSAFEFS */
 236 #undef  SOS
 237 #undef  SOX
 238 };
 239
 240 extern zone_t kalloc_zone(vm_size_t);   /* XXX */
 241
 242 /*
 243  * Initialize the kernel memory allocator
 244  */
 245 void
 246 kmeminit(void)
 247 {
 248         struct kmzones  *kmz;
 249
 250         if ((sizeof(kmzones)/sizeof(kmzones[0])) != (sizeof(memname)/sizeof(memname[0]))) {
 251                 panic("kmeminit: kmzones has %d elements but memname has %d\n",
 252                           (sizeof(kmzones)/sizeof(kmzones[0])), (sizeof(memname)/sizeof(memname[0])));
 253         }
 254
 255         kmz = kmzones;
 256         while (kmz < &kmzones[M_LAST]) {
 257 /* XXX */
 258                 if (kmz->kz_elemsize == -1)
 259                         ;
 260                 else
 261 /* XXX */
 262                 if (kmz->kz_zalloczone == KMZ_CREATEZONE) {
 263                         kmz->kz_zalloczone = zinit(kmz->kz_elemsize,
 264                                                 1024 * 1024, PAGE_SIZE,
 265                                                 memname[kmz - kmzones]);
 266                 }
 267                 else if (kmz->kz_zalloczone == KMZ_LOOKUPZONE)
 268                         kmz->kz_zalloczone = kalloc_zone(kmz->kz_elemsize);
 269
 270                 kmz++;
 271         }
 272
 273         kmz = kmzones;
 274         while (kmz < &kmzones[M_LAST]) {
 275 /* XXX */
 276                 if (kmz->kz_elemsize == -1)
 277                         ;
 278                 else
 279 /* XXX */
 280                 if (kmz->kz_zalloczone == KMZ_SHAREZONE) {
 281                         kmz->kz_zalloczone =
 282                                 kmzones[kmz->kz_elemsize].kz_zalloczone;
 283                         kmz->kz_elemsize =
 284                                 kmzones[kmz->kz_elemsize].kz_elemsize;
 285                 }
 286
 287                 kmz++;
 288         }
 289 }
 290
 291 #define MDECL(reqlen)                                   \
 292 union {                                                 \
 293         struct  _mhead hdr;                             \
 294         char    _m[(reqlen) + sizeof (struct _mhead)];  \
 295 }
 296
 297 struct _mhead {
 298         size_t  mlen;
 299         char    dat[0];
 300 };
 301
 302 #define ZEROSIZETOKEN (void *)0xFADEDFAD
 303
 304 void *
 305 _MALLOC(
 306         size_t          size,
 307         int             type,
 308         int             flags)
 309 {
 310         MDECL(size)     *mem;
 311         size_t          memsize = sizeof (*mem);
 312
 313         if (type >= M_LAST)
 314                 panic("_malloc TYPE");
 315
 316         /*
 317          * On zero request we do not return zero as that
 318          * could be mistaken for ENOMEM.
 319          */
 320         if (size == 0)
 321                 return (ZEROSIZETOKEN);
 322
 323         if (flags & M_NOWAIT) {
 324                 mem = (void *)kalloc_noblock(memsize);
 325         } else {
 326                 mem = (void *)kalloc(memsize);
 327         }
 328         if (!mem)
 329                 return (0);
 330
 331         mem->hdr.mlen = memsize;
 332
 333         if (flags & M_ZERO)
 334                 bzero(mem->hdr.dat, size);
 335
 336         return  (mem->hdr.dat);
 337 }
 338
 339 void
 340 _FREE(
 341         void            *addr,
 342         int             type)
 343 {
 344         struct _mhead   *hdr;
 345
 346         if (type >= M_LAST)
 347                 panic("_free TYPE");
 348
 349         if (addr == (void *)ZEROSIZETOKEN)
 350                 return;
 351         if (!addr)
 352                 return; /* correct (convenient bsd kernel legacy) */
 353
 354         hdr = addr; hdr--;
 355         kfree(hdr, hdr->mlen);
 356 }
 357
 358 void *
 359 _MALLOC_ZONE(
 360         size_t          size,
 361         int             type,
 362         int             flags)
 363 {
 364         struct kmzones  *kmz;
 365         void            *elem;
 366
 367         if (type >= M_LAST)
 368                 panic("_malloc_zone TYPE");
 369
 370         kmz = &kmzones[type];
 371         if (kmz->kz_zalloczone == KMZ_MALLOC)
 372                 panic("_malloc_zone ZONE: type = %d", type);
 373
 374 /* XXX */
 375         if (kmz->kz_elemsize == -1)
 376                 panic("_malloc_zone XXX");
 377 /* XXX */
 378         if (size == kmz->kz_elemsize)
 379                 if (flags & M_NOWAIT) {
 380                         elem = (void *)zalloc_noblock(kmz->kz_zalloczone);
 381                 } else {
 382                         elem = (void *)zalloc(kmz->kz_zalloczone);
 383                 }
 384         else
 385                 if (flags & M_NOWAIT) {
 386                         elem = (void *)kalloc_noblock(size);
 387                 } else {
 388                         elem = (void *)kalloc(size);
 389                 }
 390
 391         return (elem);
 392 }
 393
 394 void
 395 _FREE_ZONE(
 396         void            *elem,
 397         size_t          size,
 398         int             type)
 399 {
 400         struct kmzones  *kmz;
 401
 402         if (type >= M_LAST)
 403                 panic("FREE_SIZE");
 404
 405         kmz = &kmzones[type];
 406         if (kmz->kz_zalloczone == KMZ_MALLOC)
 407                 panic("free_zone ZONE");
 408
 409 /* XXX */
 410         if (kmz->kz_elemsize == -1)
 411                 panic("FREE_SIZE XXX");
 412 /* XXX */
 413         if (size == kmz->kz_elemsize)
 414                 zfree(kmz->kz_zalloczone, elem);
 415         else
 416                 kfree(elem, size);
 417 }