X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/2d21ac55c334faf3a56e5634905ed6987fc787d4..eee3565979933af707c711411001ba11fe406a3c:/bsd/kern/kern_malloc.c?ds=sidebyside diff --git a/bsd/kern/kern_malloc.c b/bsd/kern/kern_malloc.c index a1c8f1b50..bf2771a12 100644 --- a/bsd/kern/kern_malloc.c +++ b/bsd/kern/kern_malloc.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2013 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -74,11 +74,13 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -95,11 +97,7 @@ #include #include #include - -#include - -#include -#include +#include #include @@ -108,23 +106,21 @@ #include #include -#include - #include #include #include -void kmeminit(void) __attribute__((section("__TEXT, initcode"))); +void kmeminit(void); /* Strings corresponding to types of memory. * Must be in synch with the #defines is sys/malloc.h * NOTE - the reason we pass null strings in some cases is to reduce of foot * print as much as possible for systems where a tiny kernel is needed. - * todo - We should probably redsign this and use enums for our types and only + * todo - We should probably redesign this and use enums for our types and only * include types needed for that configuration of the kernel. This can't be * done without some kind of kpi since several types are hardwired and exported - * (for example see types M_HFSMNT, M_UDFMNT, M_TEMP, etc in sys/malloc.h) + * (for example see types M_UDFMNT, M_TEMP, etc in sys/malloc.h) */ const char *memname[] = { "free", /* 0 M_FREE */ @@ -165,11 +161,7 @@ const char *memname[] = { #else "", /* 27 M_DQUOT */ #endif -#if FFS - "UFS mount", /* 28 M_UFSMNT */ -#else - "", /* 28 M_UFSMNT */ -#endif + "proc uuid policy", /* 28 M_PROC_UUID_POLICY */ #if (SYSV_SEM || SYSV_MSG || SYSV_SHM) "shm", /* 29 M_SHM */ #else @@ -190,11 +182,7 @@ const char *memname[] = { "pstats", /* 42 M_SUBPROC */ "LFS segment", /* 43 M_SEGMENT */ "LFS node", /* 44 M_LFSNODE */ -#if FFS - "FFS node", /* 45 M_FFSNODE */ -#else "", /* 45 M_FFSNODE */ -#endif "MFS node", /* 46 M_MFSNODE */ "NQNFS Lease", /* 47 M_NQLEASE */ "NQNFS Host", /* 48 M_NQMHOST */ @@ -212,13 +200,8 @@ const char *memname[] = { "in_multi", /* 54 M_IPMADDR */ "ether_multi", /* 55 M_IFMADDR */ "mrt", /* 56 M_MRTABLE */ -#if CD9660 - "ISOFS mount", /* 57 M_ISOFSMNT */ - "ISOFS node", /* 58 M_ISOFSNODE */ -#else - "", /* 57 M_ISOFSMNT */ - "", /* 58 M_ISOFSNODE */ -#endif + "", /* 57 unused entry */ + "", /* 58 unused entry */ #if (NFSCLIENT || NFSSERVER) "NFSV3 srvdesc",/* 59 M_NFSRVDESC */ "NFSV3 diroff", /* 60 M_NFSDIROFF */ @@ -241,17 +224,11 @@ const char *memname[] = { "buf hdrs", /* 72 M_BUFHDR */ "ofile tabl", /* 73 M_OFILETABL */ "mbuf clust", /* 74 M_MCLUST */ -#if HFS - "HFS mount", /* 75 M_HFSMNT */ - "HFS node", /* 76 M_HFSNODE */ - "HFS fork", /* 77 M_HFSFORK */ -#else - "", /* 75 M_HFSMNT */ - "", /* 76 M_HFSNODE */ - "", /* 77 M_HFSFORK */ -#endif - "ZFS mount", /* 78 M_ZFSFSMNT */ - "ZFS node", /* 79 M_ZFSNODE */ + "", /* 75 unused */ + "", /* 76 unused */ + "", /* 77 unused */ + "", /* 78 unused */ + "", /* 79 unused */ "temp", /* 80 M_TEMP */ "key mgmt", /* 81 M_SECA */ "DEVFS", /* 82 M_DEVFS */ @@ -269,40 +246,65 @@ const char *memname[] = { #endif "TCP Segment Q",/* 89 M_TSEGQ */ "IGMP state", /* 90 M_IGMP */ -#if JOURNALING - "Journal", /* 91 M_JNL_JNL */ - "Transaction", /* 92 M_JNL_TR */ -#else - "", /* 91 M_JNL_JNL */ - "", /* 92 M_JNL_TR */ -#endif + "", /* 91 unused */ + "", /* 92 unused */ "specinfo", /* 93 M_SPECINFO */ "kqueue", /* 94 M_KQUEUE */ -#if HFS - "HFS dirhint", /* 95 M_HFSDIRHINT */ -#else - "", /* 95 M_HFSDIRHINT */ -#endif - "cluster_read", /* 96 M_CLRDAHEAD */ + "", /* 95 unused */ + "cluster_read", /* 96 M_CLRDAHEAD */ "cluster_write",/* 97 M_CLWRBEHIND */ "iov64", /* 98 M_IOV64 */ "fileglob", /* 99 M_FILEGLOB */ "kauth", /* 100 M_KAUTH */ "dummynet", /* 101 M_DUMMYNET */ - "unsafe_fsnode", /* 102 M_UNSAFEFS */ + "", /* 102 M_UNSAFEFS */ "macpipelabel", /* 103 M_MACPIPELABEL */ "mactemp", /* 104 M_MACTEMP */ "sbuf", /* 105 M_SBUF */ "extattr", /* 106 M_EXTATTR */ - "lctx", /* 107 M_LCTX */ + "select", /* 107 M_SELECT */ #if TRAFFIC_MGT "traffic_mgt", /* 108 M_TRAFFIC_MGT */ #else "", /* 108 M_TRAFFIC_MGT */ #endif +#if FS_COMPRESSION + "decmpfs_cnode",/* 109 M_DECMPFS_CNODE */ +#else + "", /* 109 M_DECMPFS_CNODE */ +#endif /* FS_COMPRESSION */ + "ipmfilter", /* 110 M_INMFILTER */ + "ipmsource", /* 111 M_IPMSOURCE */ + "in6mfilter", /* 112 M_IN6MFILTER */ + "ip6mopts", /* 113 M_IP6MOPTS */ + "ip6msource", /* 114 M_IP6MSOURCE */ +#if FLOW_DIVERT + "flow_divert_pcb", /* 115 M_FLOW_DIVERT_PCB */ + "flow_divert_group", /* 116 M_FLOW_DIVERT_GROUP */ +#else + "", /* 115 M_FLOW_DIVERT_PCB */ + "", /* 116 M_FLOW_DIVERT_GROUP */ +#endif + "ip6cga", /* 117 M_IP6CGA */ +#if NECP + "necp", /* 118 M_NECP */ + "necp_session_policy", /* 119 M_NECP_SESSION_POLICY */ + "necp_socket_policy", /* 120 M_NECP_SOCKET_POLICY */ + "necp_ip_policy", /* 121 M_NECP_IP_POLICY */ +#else + "", /* 118 M_NECP */ + "", /* 119 M_NECP_SESSION_POLICY */ + "", /* 120 M_NECP_SOCKET_POLICY */ + "", /* 121 M_NECP_IP_POLICY */ +#endif + "fdvnodedata" /* 122 M_FD_VN_DATA */ + "fddirbuf", /* 123 M_FD_DIRBUF */ + "netagent", /* 124 M_NETAGENT */ + "" }; /* for use with kmzones.kz_zalloczone */ +#define KMZ_CREATEZONE_ACCT ((void *)-3) #define KMZ_CREATEZONE ((void *)-2) #define KMZ_LOOKUPZONE ((void *)-1) #define KMZ_MALLOC ((void *)0) @@ -311,174 +313,178 @@ const char *memname[] = { struct kmzones { size_t kz_elemsize; void *kz_zalloczone; + boolean_t kz_noencrypt; } kmzones[M_LAST] = { #define SOS(sname) sizeof (struct sname) #define SOX(sname) -1 - { -1, 0 }, /* 0 M_FREE */ - { MSIZE, KMZ_CREATEZONE }, /* 1 M_MBUF */ - { 0, KMZ_MALLOC }, /* 2 M_DEVBUF */ - { SOS(socket), KMZ_CREATEZONE }, /* 3 M_SOCKET */ - { SOS(inpcb), KMZ_LOOKUPZONE }, /* 4 M_PCB */ - { M_MBUF, KMZ_SHAREZONE }, /* 5 M_RTABLE */ - { M_MBUF, KMZ_SHAREZONE }, /* 6 M_HTABLE */ - { M_MBUF, KMZ_SHAREZONE }, /* 7 M_FTABLE */ - { SOS(rusage), KMZ_CREATEZONE }, /* 8 M_ZOMBIE */ - { 0, KMZ_MALLOC }, /* 9 M_IFADDR */ - { M_MBUF, KMZ_SHAREZONE }, /* 10 M_SOOPTS */ - { 0, KMZ_MALLOC }, /* 11 M_SONAME */ - { MAXPATHLEN, KMZ_CREATEZONE }, /* 12 M_NAMEI */ - { 0, KMZ_MALLOC }, /* 13 M_GPROF */ - { 0, KMZ_MALLOC }, /* 14 M_IOCTLOPS */ - { 0, KMZ_MALLOC }, /* 15 M_MAPMEM */ - { SOS(ucred), KMZ_CREATEZONE }, /* 16 M_CRED */ - { SOS(pgrp), KMZ_CREATEZONE }, /* 17 M_PGRP */ - { SOS(session), KMZ_CREATEZONE }, /* 18 M_SESSION */ - { SOS(iovec_32), KMZ_LOOKUPZONE }, /* 19 M_IOV32 */ - { SOS(mount), KMZ_CREATEZONE }, /* 20 M_MOUNT */ - { 0, KMZ_MALLOC }, /* 21 M_FHANDLE */ + { -1, 0, FALSE }, /* 0 M_FREE */ + { MSIZE, KMZ_CREATEZONE, FALSE }, /* 1 M_MBUF */ + { 0, KMZ_MALLOC, FALSE }, /* 2 M_DEVBUF */ + { SOS(socket), KMZ_CREATEZONE, TRUE }, /* 3 M_SOCKET */ + { SOS(inpcb), KMZ_LOOKUPZONE, TRUE }, /* 4 M_PCB */ + { M_MBUF, KMZ_SHAREZONE, FALSE }, /* 5 M_RTABLE */ + { M_MBUF, KMZ_SHAREZONE, FALSE }, /* 6 M_HTABLE */ + { M_MBUF, KMZ_SHAREZONE, FALSE }, /* 7 M_FTABLE */ + { SOS(rusage), KMZ_CREATEZONE, TRUE }, /* 8 M_ZOMBIE */ + { 0, KMZ_MALLOC, FALSE }, /* 9 M_IFADDR */ + { M_MBUF, KMZ_SHAREZONE, FALSE }, /* 10 M_SOOPTS */ + { 0, KMZ_MALLOC, FALSE }, /* 11 M_SONAME */ + { MAXPATHLEN, KMZ_CREATEZONE, FALSE }, /* 12 M_NAMEI */ + { 0, KMZ_MALLOC, FALSE }, /* 13 M_GPROF */ + { 0, KMZ_MALLOC, FALSE }, /* 14 M_IOCTLOPS */ + { 0, KMZ_MALLOC, FALSE }, /* 15 M_MAPMEM */ + { SOS(ucred), KMZ_CREATEZONE, FALSE }, /* 16 M_CRED */ + { SOS(pgrp), KMZ_CREATEZONE, FALSE }, /* 17 M_PGRP */ + { SOS(session), KMZ_CREATEZONE, FALSE }, /* 18 M_SESSION */ + { SOS(user32_iovec), KMZ_LOOKUPZONE, FALSE },/* 19 M_IOV32 */ + { SOS(mount), KMZ_CREATEZONE, FALSE }, /* 20 M_MOUNT */ + { 0, KMZ_MALLOC, FALSE }, /* 21 M_FHANDLE */ #if (NFSCLIENT || NFSSERVER) - { SOS(nfsreq), KMZ_CREATEZONE }, /* 22 M_NFSREQ */ - { SOS(nfsmount), KMZ_CREATEZONE }, /* 23 M_NFSMNT */ - { SOS(nfsnode), KMZ_CREATEZONE }, /* 24 M_NFSNODE */ + { SOS(nfsreq), KMZ_CREATEZONE, FALSE }, /* 22 M_NFSREQ */ + { SOS(nfsmount),KMZ_CREATEZONE, FALSE }, /* 23 M_NFSMNT */ + { SOS(nfsnode), KMZ_CREATEZONE, FALSE }, /* 24 M_NFSNODE */ #else - { 0, KMZ_MALLOC }, /* 22 M_NFSREQ */ - { 0, KMZ_MALLOC }, /* 23 M_NFSMNT */ - { 0, KMZ_MALLOC }, /* 24 M_NFSNODE */ + { 0, KMZ_MALLOC, FALSE }, /* 22 M_NFSREQ */ + { 0, KMZ_MALLOC, FALSE }, /* 23 M_NFSMNT */ + { 0, KMZ_MALLOC, FALSE }, /* 24 M_NFSNODE */ #endif - { SOS(vnode), KMZ_CREATEZONE }, /* 25 M_VNODE */ - { SOS(namecache), KMZ_CREATEZONE }, /* 26 M_CACHE */ + { SOS(vnode), KMZ_CREATEZONE, TRUE }, /* 25 M_VNODE */ + { SOS(namecache), KMZ_CREATEZONE, FALSE }, /* 26 M_CACHE */ #if QUOTA - { SOX(dquot), KMZ_LOOKUPZONE }, /* 27 M_DQUOT */ + { SOX(dquot), KMZ_LOOKUPZONE, FALSE }, /* 27 M_DQUOT */ #else - { 0, KMZ_MALLOC }, /* 27 M_DQUOT */ + { 0, KMZ_MALLOC, FALSE }, /* 27 M_DQUOT */ #endif -#if FFS - { SOX(ufsmount), KMZ_LOOKUPZONE }, /* 28 M_UFSMNT */ -#else - { 0, KMZ_MALLOC }, /* 28 M_UFSMNT */ -#endif - { 0, KMZ_MALLOC }, /* 29 M_CGSUM */ - { SOS(plimit), KMZ_CREATEZONE }, /* 30 M_PLIMIT */ - { SOS(sigacts), KMZ_CREATEZONE }, /* 31 M_SIGACTS */ - { 0, KMZ_MALLOC }, /* 32 M_VMOBJ */ - { 0, KMZ_MALLOC }, /* 33 M_VMOBJHASH */ - { 0, KMZ_MALLOC }, /* 34 M_VMPMAP */ - { 0, KMZ_MALLOC }, /* 35 M_VMPVENT */ - { 0, KMZ_MALLOC }, /* 36 M_VMPAGER */ - { 0, KMZ_MALLOC }, /* 37 M_VMPGDATA */ - { SOS(fileproc), KMZ_CREATEZONE }, /* 38 M_FILEPROC */ - { SOS(filedesc), KMZ_CREATEZONE }, /* 39 M_FILEDESC */ - { SOX(lockf), KMZ_CREATEZONE }, /* 40 M_LOCKF */ - { SOS(proc), KMZ_CREATEZONE }, /* 41 M_PROC */ - { SOS(pstats), KMZ_CREATEZONE }, /* 42 M_PSTATS */ - { 0, KMZ_MALLOC }, /* 43 M_SEGMENT */ - { M_FFSNODE, KMZ_SHAREZONE }, /* 44 M_LFSNODE */ -#if FFS - { SOS(inode), KMZ_CREATEZONE }, /* 45 M_FFSNODE */ -#else - { 0, KMZ_MALLOC }, /* 45 M_FFSNODE */ -#endif - { M_FFSNODE, KMZ_SHAREZONE }, /* 46 M_MFSNODE */ - { 0, KMZ_MALLOC }, /* 47 M_NQLEASE */ - { 0, KMZ_MALLOC }, /* 48 M_NQMHOST */ - { 0, KMZ_MALLOC }, /* 49 M_NETADDR */ + { 0, KMZ_MALLOC, FALSE }, /* 28 M_PROC_UUID_POLICY */ + { 0, KMZ_MALLOC, FALSE }, /* 29 M_SHM */ + { SOS(plimit), KMZ_CREATEZONE, TRUE }, /* 30 M_PLIMIT */ + { SOS(sigacts), KMZ_CREATEZONE_ACCT, TRUE }, /* 31 M_SIGACTS */ + { 0, KMZ_MALLOC, FALSE }, /* 32 M_VMOBJ */ + { 0, KMZ_MALLOC, FALSE }, /* 33 M_VMOBJHASH */ + { 0, KMZ_MALLOC, FALSE }, /* 34 M_VMPMAP */ + { 0, KMZ_MALLOC, FALSE }, /* 35 M_VMPVENT */ + { 0, KMZ_MALLOC, FALSE }, /* 36 M_VMPAGER */ + { 0, KMZ_MALLOC, FALSE }, /* 37 M_VMPGDATA */ + { SOS(fileproc),KMZ_CREATEZONE_ACCT, TRUE }, /* 38 M_FILEPROC */ + { SOS(filedesc),KMZ_CREATEZONE_ACCT, TRUE }, /* 39 M_FILEDESC */ + { SOX(lockf), KMZ_CREATEZONE_ACCT, TRUE }, /* 40 M_LOCKF */ + { SOS(proc), KMZ_CREATEZONE, FALSE }, /* 41 M_PROC */ + { SOS(pstats), KMZ_CREATEZONE, TRUE }, /* 42 M_PSTATS */ + { 0, KMZ_MALLOC, FALSE }, /* 43 M_SEGMENT */ + { M_FFSNODE, KMZ_SHAREZONE, FALSE }, /* 44 M_LFSNODE */ + { 0, KMZ_MALLOC, FALSE }, /* 45 M_FFSNODE */ + { M_FFSNODE, KMZ_SHAREZONE, FALSE }, /* 46 M_MFSNODE */ + { 0, KMZ_MALLOC, FALSE }, /* 47 M_NQLEASE */ + { 0, KMZ_MALLOC, FALSE }, /* 48 M_NQMHOST */ + { 0, KMZ_MALLOC, FALSE }, /* 49 M_NETADDR */ #if (NFSCLIENT || NFSSERVER) { SOX(nfsrv_sock), - KMZ_CREATEZONE }, /* 50 M_NFSSVC */ - { 0, KMZ_MALLOC }, /* 51 M_NFSUID */ + KMZ_CREATEZONE_ACCT, FALSE }, /* 50 M_NFSSVC */ + { 0, KMZ_MALLOC, FALSE }, /* 51 M_NFSUID */ { SOX(nfsrvcache), - KMZ_CREATEZONE }, /* 52 M_NFSD */ + KMZ_CREATEZONE_ACCT, FALSE }, /* 52 M_NFSD */ #else - { 0, KMZ_MALLOC }, /* 50 M_NFSSVC */ - { 0, KMZ_MALLOC }, /* 51 M_NFSUID */ - { 0, KMZ_MALLOC }, /* 52 M_NFSD */ + { 0, KMZ_MALLOC, FALSE }, /* 50 M_NFSSVC */ + { 0, KMZ_MALLOC, FALSE }, /* 51 M_NFSUID */ + { 0, KMZ_MALLOC, FALSE }, /* 52 M_NFSD */ #endif { SOX(ip_moptions), - KMZ_LOOKUPZONE }, /* 53 M_IPMOPTS */ - { SOX(in_multi), KMZ_LOOKUPZONE }, /* 54 M_IPMADDR */ + KMZ_LOOKUPZONE, FALSE }, /* 53 M_IPMOPTS */ + { SOX(in_multi),KMZ_LOOKUPZONE, FALSE }, /* 54 M_IPMADDR */ { SOX(ether_multi), - KMZ_LOOKUPZONE }, /* 55 M_IFMADDR */ - { SOX(mrt), KMZ_CREATEZONE }, /* 56 M_MRTABLE */ -#if CD9660 - { SOX(iso_mnt), KMZ_LOOKUPZONE }, /* 57 M_ISOFSMNT */ - { SOS(iso_node), KMZ_CREATEZONE }, /* 58 M_ISOFSNODE */ -#else - { 0, KMZ_MALLOC }, /* 57 M_ISOFSMNT */ - { 0, KMZ_MALLOC }, /* 58 M_ISOFSNODE */ -#endif + KMZ_LOOKUPZONE, FALSE }, /* 55 M_IFMADDR */ + { SOX(mrt), KMZ_CREATEZONE, TRUE }, /* 56 M_MRTABLE */ + { 0, KMZ_MALLOC, FALSE }, /* 57 unused entry */ + { 0, KMZ_MALLOC, FALSE }, /* 58 unused entry */ #if (NFSCLIENT || NFSSERVER) { SOS(nfsrv_descript), - KMZ_CREATEZONE }, /* 59 M_NFSRVDESC */ - { SOS(nfsdmap), KMZ_CREATEZONE }, /* 60 M_NFSDIROFF */ - { SOS(fhandle), KMZ_LOOKUPZONE }, /* 61 M_NFSBIGFH */ + KMZ_CREATEZONE_ACCT, FALSE }, /* 59 M_NFSRVDESC */ + { SOS(nfsdmap), KMZ_CREATEZONE, FALSE }, /* 60 M_NFSDIROFF */ + { SOS(fhandle), KMZ_LOOKUPZONE, FALSE }, /* 61 M_NFSBIGFH */ #else - { 0, KMZ_MALLOC }, /* 59 M_NFSRVDESC */ - { 0, KMZ_MALLOC }, /* 60 M_NFSDIROFF */ - { 0, KMZ_MALLOC }, /* 61 M_NFSBIGFH */ + { 0, KMZ_MALLOC, FALSE }, /* 59 M_NFSRVDESC */ + { 0, KMZ_MALLOC, FALSE }, /* 60 M_NFSDIROFF */ + { 0, KMZ_MALLOC, FALSE }, /* 61 M_NFSBIGFH */ #endif - { 0, KMZ_MALLOC }, /* 62 M_MSDOSFSMNT */ - { 0, KMZ_MALLOC }, /* 63 M_MSDOSFSFAT */ - { 0, KMZ_MALLOC }, /* 64 M_MSDOSFSNODE */ - { SOS(tty), KMZ_CREATEZONE }, /* 65 M_TTYS */ - { 0, KMZ_MALLOC }, /* 66 M_EXEC */ - { 0, KMZ_MALLOC }, /* 67 M_MISCFSMNT */ - { 0, KMZ_MALLOC }, /* 68 M_MISCFSNODE */ - { 0, KMZ_MALLOC }, /* 69 M_ADOSFSMNT */ - { 0, KMZ_MALLOC }, /* 70 M_ADOSFSNODE */ - { 0, KMZ_MALLOC }, /* 71 M_ANODE */ - { SOX(buf), KMZ_CREATEZONE }, /* 72 M_BUFHDR */ + { 0, KMZ_MALLOC, FALSE }, /* 62 M_MSDOSFSMNT */ + { 0, KMZ_MALLOC, FALSE }, /* 63 M_MSDOSFSFAT */ + { 0, KMZ_MALLOC, FALSE }, /* 64 M_MSDOSFSNODE */ + { SOS(tty), KMZ_CREATEZONE, FALSE }, /* 65 M_TTYS */ + { 0, KMZ_MALLOC, FALSE }, /* 66 M_EXEC */ + { 0, KMZ_MALLOC, FALSE }, /* 67 M_MISCFSMNT */ + { 0, KMZ_MALLOC, FALSE }, /* 68 M_MISCFSNODE */ + { 0, KMZ_MALLOC, FALSE }, /* 69 M_ADOSFSMNT */ + { 0, KMZ_MALLOC, FALSE }, /* 70 M_ADOSFSNODE */ + { 0, KMZ_MALLOC, FALSE }, /* 71 M_ANODE */ + { 0, KMZ_MALLOC, TRUE }, /* 72 M_BUFHDR */ { (NDFILE * OFILESIZE), - KMZ_CREATEZONE }, /* 73 M_OFILETABL */ - { MCLBYTES, KMZ_CREATEZONE }, /* 74 M_MCLUST */ -#if HFS - { SOX(hfsmount), KMZ_LOOKUPZONE }, /* 75 M_HFSMNT */ - { SOS(cnode), KMZ_CREATEZONE }, /* 76 M_HFSNODE */ - { SOS(filefork), KMZ_CREATEZONE }, /* 77 M_HFSFORK */ + KMZ_CREATEZONE_ACCT, FALSE }, /* 73 M_OFILETABL */ + { MCLBYTES, KMZ_CREATEZONE, FALSE }, /* 74 M_MCLUST */ + { 0, KMZ_MALLOC, FALSE }, /* 75 unused */ + { 0, KMZ_MALLOC, FALSE }, /* 76 unused */ + { 0, KMZ_MALLOC, FALSE }, /* 77 unused */ + { 0, KMZ_MALLOC, FALSE }, /* 78 unused */ + { 0, KMZ_MALLOC, FALSE }, /* 79 unused */ + { 0, KMZ_MALLOC, FALSE }, /* 80 M_TEMP */ + { 0, KMZ_MALLOC, FALSE }, /* 81 M_SECA */ + { 0, KMZ_MALLOC, FALSE }, /* 82 M_DEVFS */ + { 0, KMZ_MALLOC, FALSE }, /* 83 M_IPFW */ + { 0, KMZ_MALLOC, FALSE }, /* 84 M_UDFNODE */ + { 0, KMZ_MALLOC, FALSE }, /* 85 M_UDFMOUNT */ + { 0, KMZ_MALLOC, FALSE }, /* 86 M_IP6NDP */ + { 0, KMZ_MALLOC, FALSE }, /* 87 M_IP6OPT */ + { 0, KMZ_MALLOC, FALSE }, /* 88 M_IP6MISC */ + { 0, KMZ_MALLOC, FALSE }, /* 89 M_TSEGQ */ + { 0, KMZ_MALLOC, FALSE }, /* 90 M_IGMP */ + { 0, KMZ_MALLOC, FALSE }, /* 91 unused */ + { 0, KMZ_MALLOC, FALSE }, /* 92 unused */ + { SOS(specinfo),KMZ_CREATEZONE, TRUE }, /* 93 M_SPECINFO */ + { SOS(kqueue), KMZ_CREATEZONE, FALSE }, /* 94 M_KQUEUE */ + { 0, KMZ_MALLOC, FALSE }, /* 95 unused */ + { SOS(cl_readahead), KMZ_CREATEZONE, TRUE }, /* 96 M_CLRDAHEAD */ + { SOS(cl_writebehind),KMZ_CREATEZONE, TRUE }, /* 97 M_CLWRBEHIND */ + { SOS(user64_iovec), KMZ_LOOKUPZONE, FALSE },/* 98 M_IOV64 */ + { SOS(fileglob), KMZ_CREATEZONE, TRUE }, /* 99 M_FILEGLOB */ + { 0, KMZ_MALLOC, FALSE }, /* 100 M_KAUTH */ + { 0, KMZ_MALLOC, FALSE }, /* 101 M_DUMMYNET */ + { 0, KMZ_MALLOC, FALSE }, /* 102 M_UNSAFEFS */ + { 0, KMZ_MALLOC, FALSE }, /* 103 M_MACPIPELABEL */ + { 0, KMZ_MALLOC, FALSE }, /* 104 M_MACTEMP */ + { 0, KMZ_MALLOC, FALSE }, /* 105 M_SBUF */ + { 0, KMZ_MALLOC, FALSE }, /* 106 M_HFS_EXTATTR */ + { 0, KMZ_MALLOC, FALSE }, /* 107 M_SELECT */ + { 0, KMZ_MALLOC, FALSE }, /* 108 M_TRAFFIC_MGT */ +#if FS_COMPRESSION + { SOS(decmpfs_cnode),KMZ_CREATEZONE , FALSE}, /* 109 M_DECMPFS_CNODE */ #else - { 0, KMZ_MALLOC }, /* 75 M_HFSMNT */ - { 0, KMZ_MALLOC }, /* 76 M_HFSNODE */ - { 0, KMZ_MALLOC }, /* 77 M_HFSFORK */ -#endif - { 0, KMZ_MALLOC }, /* 78 M_ZFSMNT */ - { 0, KMZ_MALLOC }, /* 79 M_ZFSNODE */ - { 0, KMZ_MALLOC }, /* 80 M_TEMP */ - { 0, KMZ_MALLOC }, /* 81 M_SECA */ - { 0, KMZ_MALLOC }, /* 82 M_DEVFS */ - { 0, KMZ_MALLOC }, /* 83 M_IPFW */ - { 0, KMZ_MALLOC }, /* 84 M_UDFNODE */ - { 0, KMZ_MALLOC }, /* 85 M_UDFMOUNT */ - { 0, KMZ_MALLOC }, /* 86 M_IP6NDP */ - { 0, KMZ_MALLOC }, /* 87 M_IP6OPT */ - { 0, KMZ_MALLOC }, /* 88 M_IP6MISC */ - { 0, KMZ_MALLOC }, /* 89 M_TSEGQ */ - { 0, KMZ_MALLOC }, /* 90 M_IGMP */ -#if JOURNALING - { SOS(journal), KMZ_CREATEZONE }, /* 91 M_JNL_JNL */ - { SOS(transaction), KMZ_CREATEZONE }, /* 92 M_JNL_TR */ + { 0, KMZ_MALLOC, FALSE }, /* 109 M_DECMPFS_CNODE */ +#endif /* FS_COMPRESSION */ + { 0, KMZ_MALLOC, FALSE }, /* 110 M_INMFILTER */ + { 0, KMZ_MALLOC, FALSE }, /* 111 M_IPMSOURCE */ + { 0, KMZ_MALLOC, FALSE }, /* 112 M_IN6MFILTER */ + { 0, KMZ_MALLOC, FALSE }, /* 113 M_IP6MOPTS */ + { 0, KMZ_MALLOC, FALSE }, /* 114 M_IP6MSOURCE */ +#if FLOW_DIVERT + { SOS(flow_divert_pcb), KMZ_CREATEZONE, TRUE }, /* 115 M_FLOW_DIVERT_PCB */ + { SOS(flow_divert_group), KMZ_CREATEZONE, TRUE }, /* 116 M_FLOW_DIVERT_GROUP */ #else - { 0, KMZ_MALLOC }, /* 91 M_JNL_JNL */ - { 0, KMZ_MALLOC }, /* 92 M_JNL_TR */ -#endif - { SOS(specinfo), KMZ_CREATEZONE }, /* 93 M_SPECINFO */ - { SOS(kqueue), KMZ_CREATEZONE }, /* 94 M_KQUEUE */ -#if HFS - { SOS(directoryhint), KMZ_CREATEZONE }, /* 95 M_HFSDIRHINT */ + { 0, KMZ_MALLOC, FALSE }, /* 115 M_FLOW_DIVERT_PCB */ + { 0, KMZ_MALLOC, FALSE }, /* 116 M_FLOW_DIVERT_GROUP */ +#endif /* FLOW_DIVERT */ + { 0, KMZ_MALLOC, FALSE }, /* 117 M_IP6CGA */ + { 0, KMZ_MALLOC, FALSE }, /* 118 M_NECP */ +#if NECP + { SOS(necp_session_policy), KMZ_CREATEZONE, TRUE }, /* 119 M_NECP_SESSION_POLICY */ + { SOS(necp_kernel_socket_policy), KMZ_CREATEZONE, TRUE }, /* 120 M_NECP_SOCKET_POLICY */ + { SOS(necp_kernel_ip_output_policy), KMZ_CREATEZONE, TRUE }, /* 121 M_NECP_IP_POLICY */ #else - { 0, KMZ_MALLOC }, /* 95 M_HFSDIRHINT */ -#endif - { SOS(cl_readahead), KMZ_CREATEZONE }, /* 96 M_CLRDAHEAD */ - { SOS(cl_writebehind),KMZ_CREATEZONE }, /* 97 M_CLWRBEHIND */ - { SOS(iovec_64), KMZ_LOOKUPZONE }, /* 98 M_IOV64 */ - { SOS(fileglob), KMZ_CREATEZONE }, /* 99 M_FILEGLOB */ - { 0, KMZ_MALLOC }, /* 100 M_KAUTH */ - { 0, KMZ_MALLOC }, /* 101 M_DUMMYNET */ - { SOS(unsafe_fsnode),KMZ_CREATEZONE }, /* 102 M_UNSAFEFS */ - { 0, KMZ_MALLOC }, /* 103 M_MACPIPELABEL */ - { 0, KMZ_MALLOC }, /* 104 M_MACTEMP */ - { 0, KMZ_MALLOC }, /* 105 M_SBUF */ - { 0, KMZ_MALLOC }, /* 106 M_HFS_EXTATTR */ - { 0, KMZ_MALLOC }, /* 107 M_LCTX */ - { 0, KMZ_MALLOC }, /* 108 M_TRAFFIC_MGT */ + { 0, KMZ_MALLOC, FALSE }, /* 119 M_NECP_SESSION_POLICY */ + { 0, KMZ_MALLOC, FALSE }, /* 120 M_NECP_SOCKET_POLICY */ + { 0, KMZ_MALLOC, FALSE }, /* 121 M_NECP_IP_POLICY */ +#endif /* NECP */ + { 0, KMZ_MALLOC, FALSE }, /* 122 M_FD_VN_DATA */ + { 0, KMZ_MALLOC, FALSE }, /* 123 M_FD_DIRBUF */ + { 0, KMZ_MALLOC, FALSE }, /* 124 M_NETAGENT */ #undef SOS #undef SOX }; @@ -505,10 +511,16 @@ kmeminit(void) ; else /* XXX */ - if (kmz->kz_zalloczone == KMZ_CREATEZONE) { + if (kmz->kz_zalloczone == KMZ_CREATEZONE || + kmz->kz_zalloczone == KMZ_CREATEZONE_ACCT) { kmz->kz_zalloczone = zinit(kmz->kz_elemsize, 1024 * 1024, PAGE_SIZE, memname[kmz - kmzones]); + zone_change(kmz->kz_zalloczone, Z_CALLERACCT, + (kmz->kz_zalloczone == KMZ_CREATEZONE_ACCT)); + + if (kmz->kz_noencrypt == TRUE) + zone_change(kmz->kz_zalloczone, Z_NOENCRYPT, TRUE); } else if (kmz->kz_zalloczone == KMZ_LOOKUPZONE) kmz->kz_zalloczone = kalloc_zone(kmz->kz_elemsize); @@ -534,25 +546,30 @@ kmeminit(void) } } -#define MDECL(reqlen) \ -union { \ - struct _mhead hdr; \ - char _m[(reqlen) + sizeof (struct _mhead)]; \ +void * +_MALLOC_external( + size_t size, + int type, + int flags); +void * +_MALLOC_external( + size_t size, + int type, + int flags) +{ + static vm_allocation_site_t site = { VM_KERN_MEMORY_KALLOC, VM_TAG_BT }; + return (__MALLOC(size, type, flags, &site)); } -struct _mhead { - size_t mlen; - char dat[0]; -}; - void * -_MALLOC( +__MALLOC( size_t size, int type, - int flags) + int flags, + vm_allocation_site_t *site) { - MDECL(size) *mem; - size_t memsize = sizeof (*mem); + void *addr = NULL; + vm_size_t msize = size; if (type >= M_LAST) panic("_malloc TYPE"); @@ -560,20 +577,35 @@ _MALLOC( if (size == 0) return (NULL); + if (msize != size) { + panic("Requested size to __MALLOC is too large (%llx)!\n", (uint64_t)size); + } + if (flags & M_NOWAIT) { - mem = (void *)kalloc_noblock(memsize); + addr = (void *)kalloc_canblock(&msize, FALSE, site); } else { - mem = (void *)kalloc(memsize); + addr = (void *)kalloc_canblock(&msize, TRUE, site); + if (addr == NULL) { + /* + * We get here when the caller told us to block waiting for memory, but + * kalloc said there's no memory left to get. Generally, this means there's a + * leak or the caller asked for an impossibly large amount of memory. Since there's + * nothing left to wait for and the caller isn't expecting a NULL return code, we + * just panic. This is less than ideal, but returning NULL doesn't help since the + * majority of callers don't check the return value and will just dereference the pointer and + * trap anyway. We may as well get a more descriptive message out while we can. + */ + + panic("_MALLOC: kalloc returned NULL (potential leak), size %llu", (uint64_t) size); + } } - if (!mem) + if (!addr) return (0); - mem->hdr.mlen = memsize; - if (flags & M_ZERO) - bzero(mem->hdr.dat, size); + bzero(addr, size); - return (mem->hdr.dat); + return (addr); } void @@ -581,23 +613,70 @@ _FREE( void *addr, int type) { - struct _mhead *hdr; - if (type >= M_LAST) panic("_free TYPE"); if (!addr) return; /* correct (convenient bsd kernel legacy) */ - hdr = addr; hdr--; - kfree(hdr, hdr->mlen); + kfree_addr(addr); } void * -_MALLOC_ZONE( +__REALLOC( + void *addr, + size_t size, + int type, + int flags, + vm_allocation_site_t *site) +{ + void *newaddr; + size_t alloc; + + /* realloc(NULL, ...) is equivalent to malloc(...) */ + if (addr == NULL) + return (__MALLOC(size, type, flags, site)); + + alloc = kalloc_size(addr); + /* + * Find out the size of the bucket in which the new sized allocation + * would land. If it matches the bucket of the original allocation, + * simply return the address. + */ + if (kalloc_bucket_size(size) == alloc) + return addr; + + /* Allocate a new, bigger (or smaller) block */ + if ((newaddr = __MALLOC(size, type, flags, site)) == NULL) + return (NULL); + + /* Copy over original contents */ + bcopy(addr, newaddr, MIN(size, alloc)); + _FREE(addr, type); + + return (newaddr); +} + +void * +_MALLOC_ZONE_external( + size_t size, + int type, + int flags); +void * +_MALLOC_ZONE_external( size_t size, int type, int flags) +{ + return (__MALLOC_ZONE(size, type, flags, NULL)); +} + +void * +__MALLOC_ZONE( + size_t size, + int type, + int flags, + vm_allocation_site_t *site) { struct kmzones *kmz; void *elem; @@ -619,12 +698,16 @@ _MALLOC_ZONE( } else { elem = (void *)zalloc(kmz->kz_zalloczone); } - else - if (flags & M_NOWAIT) { - elem = (void *)kalloc_noblock(size); + else { + vm_size_t kalloc_size = size; + if (size > kalloc_size) { + elem = NULL; + } else if (flags & M_NOWAIT) { + elem = (void *)kalloc_canblock(&kalloc_size, FALSE, site); } else { - elem = (void *)kalloc(size); + elem = (void *)kalloc_canblock(&kalloc_size, TRUE, site); } + } return (elem); } @@ -653,3 +736,116 @@ _FREE_ZONE( else kfree(elem, size); } + +#if CONFIG_ZLEAKS + +SYSCTL_DECL(_kern_zleak); +SYSCTL_NODE(_kern, OID_AUTO, zleak, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "zleak"); + +/* + * kern.zleak.active + * + * Show the status of the zleak subsystem (0 = enabled, 1 = active, + * and -1 = failed), and if enabled, allow it to be activated immediately. + */ +static int +sysctl_zleak_active SYSCTL_HANDLER_ARGS +{ +#pragma unused(arg1, arg2) + int oldval, val, error; + + val = oldval = get_zleak_state(); + error = sysctl_handle_int(oidp, &val, 0, req); + if (error || !req->newptr) + return (error); + /* + * Can only be activated if it's off (and not failed.) + * Cannot be deactivated once it's on. + */ + if (val == 1 && oldval == 0) { + kern_return_t kr = zleak_activate(); + + if (KERN_SUCCESS != kr) + printf("zleak_active: failed to activate " + "live zone leak debugging (%d).\n", kr); + } if (val == 0 && oldval == 1) { + printf("zleak_active: active, cannot be disabled.\n"); + return (EINVAL); + } + return (0); +} + +SYSCTL_PROC(_kern_zleak, OID_AUTO, active, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + 0, 0, sysctl_zleak_active, "I", "zleak activity"); + +/* + * kern.zleak.max_zonemap_size + * + * Read the value of the maximum zonemap size in bytes; useful + * as the maximum size that zleak.global_threshold and + * zleak.zone_threshold should be set to. + */ +static int +sysctl_zleak_max_zonemap_size SYSCTL_HANDLER_ARGS +{ + uint64_t zmap_max_size = *(vm_size_t *)arg1; + + return sysctl_handle_quad(oidp, &zmap_max_size, arg2, req); +} + +SYSCTL_PROC(_kern_zleak, OID_AUTO, max_zonemap_size, + CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, + &zleak_max_zonemap_size, 0, + sysctl_zleak_max_zonemap_size, "Q", "zleak max zonemap size"); + + +static int +sysctl_zleak_threshold SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg2) + int error; + uint64_t value = *(vm_size_t *)arg1; + + error = sysctl_io_number(req, value, sizeof (value), &value, NULL); + + if (error || !req->newptr) + return (error); + + if (value > (uint64_t)zleak_max_zonemap_size) + return (ERANGE); + + *(vm_size_t *)arg1 = value; + return (0); +} + +/* + * kern.zleak.global_threshold + * + * Set the global zleak threshold size (in bytes). If the zone map + * grows larger than this value, zleaks are automatically activated. + * + * The default value is set in zleak_init(). + */ +SYSCTL_PROC(_kern_zleak, OID_AUTO, global_threshold, + CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, + &zleak_global_tracking_threshold, 0, + sysctl_zleak_threshold, "Q", "zleak global threshold"); + +/* + * kern.zleak.zone_threshold + * + * Set the per-zone threshold size (in bytes) above which any + * zone will automatically start zleak tracking. + * + * The default value is set in zleak_init(). + * + * Setting this variable will have no effect until zleak tracking is + * activated (See above.) + */ +SYSCTL_PROC(_kern_zleak, OID_AUTO, zone_threshold, + CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, + &zleak_per_zone_tracking_threshold, 0, + sysctl_zleak_threshold, "Q", "zleak per-zone threshold"); + +#endif /* CONFIG_ZLEAKS */