xnu-2050.7.9.tar.gz

[apple/xnu.git] / osfmk / kern / kalloc.c

diff --git a/osfmk/kern/kalloc.c b/osfmk/kern/kalloc.c
index 4f21c6008652f988e697da1238666a0aa1439418..a5febf94c7fa14740329e8676b38760f76700d6e 100644 (file)
--- a/osfmk/kern/kalloc.c
+++ b/osfmk/kern/kalloc.c
@@ -1,16 +1,19 @@
 /*
 /*

- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2011 Apple Computer, Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
- * 
- * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in

- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.

  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER


@@ -20,166 +23,11 @@
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /*
  * @OSF_COPYRIGHT@
  */
  */
 /*
  * @OSF_COPYRIGHT@
  */

-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:34  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:55  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.2.19.5  1995/02/24  15:20:29  alanl
- *     Lock package cleanup.
- *     [95/02/15            alanl]
- *
- *     Merge with DIPC2_SHARED.
- *     [1995/01/05  15:11:02  alanl]
- *
- * Revision 1.2.28.2  1994/11/10  06:12:50  dwm
- *     mk6 CR764 - s/spinlock/simple_lock/ (name change only)
- *     [1994/11/10  05:28:35  dwm]
- * 
- * Revision 1.2.28.1  1994/11/04  10:07:40  dwm
- *     mk6 CR668 - 1.3b26 merge
- *     * Revision 1.2.2.4  1993/11/08  15:04:18  gm
- *     CR9710: Updated to new zinit() and zone_change() interfaces.
- *     * End1.3merge
- *     [1994/11/04  09:25:48  dwm]
- * 
- * Revision 1.2.19.3  1994/09/23  02:20:52  ezf
- *     change marker to not FREE
- *     [1994/09/22  21:33:57  ezf]
- * 
- * Revision 1.2.19.2  1994/06/14  18:36:36  bolinger
- *     NMK17.2 merge:  Replace simple_lock ops.
- *     [1994/06/14  18:35:17  bolinger]
- * 
- * Revision 1.2.19.1  1994/06/14  17:04:23  bolinger
- *     Merge up to NMK17.2.
- *     [1994/06/14  16:54:19  bolinger]
- * 
- * Revision 1.2.23.3  1994/10/14  12:24:33  sjs
- *     Removed krealloc_spinl routine: the newer locking scheme makes it
- *     obsolete.
- *     [94/10/13            sjs]
- * 
- * Revision 1.2.23.2  1994/08/11  14:42:46  rwd
- *     Post merge cleanup
- *     [94/08/09            rwd]
- * 
- *     Changed zcollectable to use zchange.
- *     [94/08/04            rwd]
- * 
- * Revision 1.2.17.2  1994/07/08  01:58:45  alanl
- *     Change comment to match function name.
- *     [1994/07/08  01:47:59  alanl]
- * 
- * Revision 1.2.17.1  1994/05/26  16:20:38  sjs
- *     Added krealloc_spinl: same as krealloc but uses spin locks.
- *     [94/05/25            sjs]
- * 
- * Revision 1.2.23.1  1994/08/04  02:24:55  mmp
- *     Added krealloc_spinl: same as krealloc but uses spin locks.
- *     [94/05/25            sjs]
- * 
- * Revision 1.2.13.1  1994/02/11  14:27:12  paire
- *     Changed krealloc() to make it work on a MP system. Added a new parameter
- *     which is the simple lock that should be held while modifying the memory
- *     area already initialized.
- *     Change from NMK16.1 [93/09/02            paire]
- * 
- *     Do not set debug for kalloc zones as default. It wastes
- *     to much space.
- *     Change from NMK16.1 [93/08/16            bernadat]
- *     [94/02/07            paire]
- * 
- * Revision 1.2.2.3  1993/07/28  17:15:44  bernard
- *     CR9523 -- Prototypes.
- *     [1993/07/27  20:14:12  bernard]
- * 
- * Revision 1.2.2.2  1993/06/02  23:37:46  jeffc
- *     Added to OSF/1 R1.3 from NMK15.0.
- *     [1993/06/02  21:12:59  jeffc]
- * 
- * Revision 1.2  1992/12/07  21:28:42  robert
- *     integrate any changes below for 14.0 (branch from 13.16 base)
- * 
- *     Joseph Barrera (jsb) at Carnegie-Mellon University 11-Sep-92
- *     Added krealloc. Added kalloc_max_prerounded for quicker choice between
- *     zalloc and kmem_alloc. Renamed MINSIZE to KALLOC_MINSIZE.
- *     [1992/12/06  19:47:16  robert]
- * 
- * Revision 1.1  1992/09/30  02:09:23  robert
- *     Initial revision
- * 
- * $EndLog$
- */
-/* CMU_HIST */
-/*
- * Revision 2.9  91/05/14  16:43:17  mrt
- *     Correcting copyright
- * 
- * Revision 2.8  91/03/16  14:50:37  rpd
- *     Updated for new kmem_alloc interface.
- *     [91/03/03            rpd]
- * 
- * Revision 2.7  91/02/05  17:27:22  mrt
- *     Changed to new Mach copyright
- *     [91/02/01  16:14:12  mrt]
- * 
- * Revision 2.6  90/06/19  22:59:06  rpd
- *     Made the big kalloc zones collectable.
- *     [90/06/05            rpd]
- * 
- * Revision 2.5  90/06/02  14:54:47  rpd
- *     Added kalloc_max, kalloc_map_size.
- *     [90/03/26  22:06:39  rpd]
- * 
- * Revision 2.4  90/01/11  11:43:13  dbg
- *     De-lint.
- *     [89/12/06            dbg]
- * 
- * Revision 2.3  89/09/08  11:25:51  dbg
- *     MACH_KERNEL: remove non-MACH data types.
- *     [89/07/11            dbg]
- * 
- * Revision 2.2  89/08/31  16:18:59  rwd
- *     First Checkin
- *     [89/08/23  15:41:37  rwd]
- * 
- * Revision 2.6  89/08/02  08:03:28  jsb
- *     Make all kalloc zones 8 MB big. (No more kalloc panics!)
- *     [89/08/01  14:10:17  jsb]
- * 
- * Revision 2.4  89/04/05  13:03:10  rvb
- *     Guarantee a zone max of at least 100 elements or 10 pages
- *     which ever is greater.  Afs (AllocDouble()) puts a great demand
- *     on the 2048 zone and used to blow away.
- *     [89/03/09            rvb]
- * 
- * Revision 2.3  89/02/25  18:04:39  gm0w
- *     Changes for cleanup.
- * 
- * Revision 2.2  89/01/18  02:07:04  jsb
- *     Give each kalloc zone a meaningful name (for panics);
- *     create a zone for each power of 2 between MINSIZE
- *     and PAGE_SIZE, instead of using (obsoleted) NQUEUES.
- *     [89/01/17  10:16:33  jsb]
- * 
- *
- * 13-Feb-88  John Seamons (jks) at NeXT
- *     Updated to use kmem routines instead of vmem routines.
- *
- * 21-Jun-85  Avadis Tevanian (avie) at Carnegie-Mellon University
- *     Created.
- */
-/* CMU_ENDHIST */

 /* 
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
 /* 
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University


@@ -225,49 +73,212 @@
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 #include <kern/lock.h>
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 #include <kern/lock.h>

+#include <kern/ledger.h>

 #include <vm/vm_kern.h>
 #include <vm/vm_object.h>
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_object.h>
 #include <vm/vm_map.h>

+#include <libkern/OSMalloc.h>

 
 #ifdef MACH_BSD
 zone_t kalloc_zone(vm_size_t);
 #endif
 
 
 #ifdef MACH_BSD
 zone_t kalloc_zone(vm_size_t);
 #endif
 

+#define KALLOC_MAP_SIZE_MIN  (16 * 1024 * 1024)
+#define KALLOC_MAP_SIZE_MAX  (128 * 1024 * 1024)

 vm_map_t kalloc_map;
 vm_map_t kalloc_map;

-vm_size_t kalloc_map_size = 16 * 1024 * 1024;

 vm_size_t kalloc_max;
 vm_size_t kalloc_max_prerounded;
 vm_size_t kalloc_max;
 vm_size_t kalloc_max_prerounded;

+vm_size_t kalloc_kernmap_size; /* size of kallocs that can come from kernel map */

 
 unsigned int kalloc_large_inuse;
 vm_size_t    kalloc_large_total;
 vm_size_t    kalloc_large_max;
 
 unsigned int kalloc_large_inuse;
 vm_size_t    kalloc_large_total;
 vm_size_t    kalloc_large_max;

+vm_size_t    kalloc_largest_allocated = 0;
+uint64_t    kalloc_large_sum;
+
+int    kalloc_fake_zone_index = -1; /* index of our fake zone in statistics arrays */
+
+vm_offset_t    kalloc_map_min;
+vm_offset_t    kalloc_map_max;
+
+#ifdef MUTEX_ZONE
+/*
+ * Diagnostic code to track mutexes separately rather than via the 2^ zones
+ */
+       zone_t          lck_mtx_zone;
+#endif
+
+static void
+KALLOC_ZINFO_SALLOC(vm_size_t bytes)
+{
+       thread_t thr = current_thread();
+       task_t task;
+       zinfo_usage_t zinfo;
+
+       ledger_debit(thr->t_ledger, task_ledgers.tkm_shared, bytes);
+
+       if (kalloc_fake_zone_index != -1 && 
+           (task = thr->task) != NULL && (zinfo = task->tkm_zinfo) != NULL)
+               zinfo[kalloc_fake_zone_index].alloc += bytes;
+}
+
+static void
+KALLOC_ZINFO_SFREE(vm_size_t bytes)
+{
+       thread_t thr = current_thread();
+       task_t task;
+       zinfo_usage_t zinfo;
+
+       ledger_credit(thr->t_ledger, task_ledgers.tkm_shared, bytes);
+
+       if (kalloc_fake_zone_index != -1 && 
+           (task = thr->task) != NULL && (zinfo = task->tkm_zinfo) != NULL)
+               zinfo[kalloc_fake_zone_index].free += bytes;
+}

 
 /*
  *     All allocations of size less than kalloc_max are rounded to the
 
 /*
  *     All allocations of size less than kalloc_max are rounded to the

- *     next highest power of 2.  This allocator is built on top of
+ *     next nearest sized zone.  This allocator is built on top of

  *     the zone allocator.  A zone is created for each potential size
  *     that we are willing to get in small blocks.
  *
  *     We assume that kalloc_max is not greater than 64K;
  *     the zone allocator.  A zone is created for each potential size
  *     that we are willing to get in small blocks.
  *
  *     We assume that kalloc_max is not greater than 64K;

- *     thus 16 is a safe array size for k_zone and k_zone_name.

  *
  *     Note that kalloc_max is somewhat confusingly named.
  *     It represents the first power of two for which no zone exists.
  *     kalloc_max_prerounded is the smallest allocation size, before
  *     rounding, for which no zone exists.
  *
  *     Note that kalloc_max is somewhat confusingly named.
  *     It represents the first power of two for which no zone exists.
  *     kalloc_max_prerounded is the smallest allocation size, before
  *     rounding, for which no zone exists.

+ *
+ *     Also if the allocation size is more than kalloc_kernmap_size 
+ *     then allocate from kernel map rather than kalloc_map.
+ */
+
+#if KALLOC_MINSIZE == 16 && KALLOC_LOG2_MINALIGN == 4
+
+/*
+ * "Legacy" aka "power-of-2" backing zones with 16-byte minimum
+ * size and alignment.  Users of this profile would probably
+ * benefit from some tuning.
+ */
+
+#define K_ZONE_SIZES                   \
+       16,                             \
+       32,                             \
+/* 6 */        64,                             \
+       128,                            \
+       256,                            \
+/* 9 */        512,                            \
+       1024,                           \
+       2048,                           \
+/* C */        4096
+
+
+#define K_ZONE_NAMES                   \
+       "kalloc.16",                    \
+       "kalloc.32",                    \
+/* 6 */        "kalloc.64",                    \
+       "kalloc.128",                   \
+       "kalloc.256",                   \
+/* 9 */        "kalloc.512",                   \
+       "kalloc.1024",                  \
+       "kalloc.2048",                  \
+/* C */        "kalloc.4096"
+
+#define K_ZONE_MAXIMA                  \
+       1024,                           \
+       4096,                           \
+/* 6 */        4096,                           \
+       4096,                           \
+       4096,                           \
+/* 9 */        1024,                           \
+       1024,                           \
+       1024,                           \
+/* C */        1024
+
+#elif KALLOC_MINSIZE == 8 && KALLOC_LOG2_MINALIGN == 3
+
+/*
+ * Tweaked for ARM (and x64) in 04/2011

  */
 
  */
 

-int first_k_zone = -1;
-struct zone *k_zone[16];
-static char *k_zone_name[16] = {
-       "kalloc.1",             "kalloc.2",
-       "kalloc.4",             "kalloc.8",
-       "kalloc.16",            "kalloc.32",
-       "kalloc.64",            "kalloc.128",
-       "kalloc.256",           "kalloc.512",
-       "kalloc.1024",          "kalloc.2048",
-       "kalloc.4096",          "kalloc.8192",
-       "kalloc.16384",         "kalloc.32768"
+#define K_ZONE_SIZES                   \
+/* 3 */        8,                              \
+       16,     24,                     \
+       32,     40,     48,             \
+/* 6 */        64,     88,     112,            \
+       128,    192,                    \
+       256,    384,                    \
+/* 9 */        512,    768,                    \
+       1024,   1536,                   \
+       2048,   3072,                   \
+       4096,   6144
+
+#define K_ZONE_NAMES                   \
+/* 3 */        "kalloc.8",                     \
+       "kalloc.16",    "kalloc.24",    \
+       "kalloc.32",    "kalloc.40",    "kalloc.48",    \
+/* 6 */        "kalloc.64",    "kalloc.88",    "kalloc.112",   \
+       "kalloc.128",   "kalloc.192",   \
+       "kalloc.256",   "kalloc.384",   \
+/* 9 */        "kalloc.512",   "kalloc.768",   \
+       "kalloc.1024",  "kalloc.1536",  \
+       "kalloc.2048",  "kalloc.3072",  \
+       "kalloc.4096",  "kalloc.6144"
+
+#define        K_ZONE_MAXIMA                   \
+/* 3 */        1024,                           \
+       1024,   1024,                   \
+       4096,   4096,   4096,           \
+/* 6 */        4096,   4096,   4096,           \
+       4096,   4096,                   \
+       4096,   4096,                   \
+/* 9 */        1024,   1024,                   \
+       1024,   1024,                   \
+       1024,   1024,                   \
+/* C */        1024,   64
+
+#else
+#error missing zone size parameters for kalloc
+#endif
+
+#define KALLOC_MINALIGN (1 << KALLOC_LOG2_MINALIGN)
+
+static const int k_zone_size[] = {
+       K_ZONE_SIZES,
+       8192,
+       16384,
+/* F */        32768
+};
+
+#define N_K_ZONE       (sizeof (k_zone_size) / sizeof (k_zone_size[0]))
+
+/*
+ * Many kalloc() allocations are for small structures containing a few
+ * pointers and longs - the k_zone_dlut[] direct lookup table, indexed by
+ * size normalized to the minimum alignment, finds the right zone index
+ * for them in one dereference.
+ */
+
+#define INDEX_ZDLUT(size)      \
+                       (((size) + KALLOC_MINALIGN - 1) / KALLOC_MINALIGN)
+#define N_K_ZDLUT      (2048 / KALLOC_MINALIGN)
+                               /* covers sizes [0 .. 2048 - KALLOC_MINALIGN] */
+#define MAX_SIZE_ZDLUT ((N_K_ZDLUT - 1) * KALLOC_MINALIGN)
+
+static int8_t k_zone_dlut[N_K_ZDLUT];  /* table of indices into k_zone[] */
+
+/*
+ * If there's no hit in the DLUT, then start searching from k_zindex_start.
+ */
+static int k_zindex_start;
+
+static zone_t k_zone[N_K_ZONE];
+
+static const char *k_zone_name[N_K_ZONE] = {
+       K_ZONE_NAMES,
+       "kalloc.8192",
+       "kalloc.16384",
+/* F */        "kalloc.32768"

 };
 
 /*
 };
 
 /*


@@ -276,25 +287,44 @@ static char *k_zone_name[16] = {
  *  based on need, rather than just guessing; it also
  *  means its patchable in case you're wrong!
  */
  *  based on need, rather than just guessing; it also
  *  means its patchable in case you're wrong!
  */

-unsigned long k_zone_max[16] = {
-      1024,            /*      1 Byte  */
-      1024,            /*      2 Byte  */
-      1024,            /*      4 Byte  */
-      1024,            /*      8 Byte  */
-      1024,            /*     16 Byte  */
-      4096,            /*     32 Byte  */
-      4096,            /*     64 Byte  */
-      4096,            /*    128 Byte  */
-      4096,            /*    256 Byte  */
-      1024,            /*    512 Byte  */
-      1024,            /*   1024 Byte  */
-      1024,            /*   2048 Byte  */
-      1024,            /*   4096 Byte  */
-      4096,            /*   8192 Byte  */
-      64,              /*  16384 Byte  */
-      64,              /*  32768 Byte  */
+unsigned int k_zone_max[N_K_ZONE] = {
+       K_ZONE_MAXIMA,
+       4096,
+       64,
+/* F */        64

 };
 
 };
 

+/* #define KALLOC_DEBUG                1 */
+
+/* forward declarations */
+void * kalloc_canblock(
+               vm_size_t       size,
+               boolean_t       canblock);
+
+
+lck_grp_t *kalloc_lck_grp;
+lck_mtx_t kalloc_lock;
+
+#define kalloc_spin_lock()     lck_mtx_lock_spin(&kalloc_lock)
+#define kalloc_unlock()                lck_mtx_unlock(&kalloc_lock)
+
+
+/* OSMalloc local data declarations */
+static
+queue_head_t    OSMalloc_tag_list;
+
+lck_grp_t *OSMalloc_tag_lck_grp;
+lck_mtx_t OSMalloc_tag_lock;
+
+#define OSMalloc_tag_spin_lock()       lck_mtx_lock_spin(&OSMalloc_tag_lock)
+#define OSMalloc_tag_unlock()          lck_mtx_unlock(&OSMalloc_tag_lock)
+
+
+/* OSMalloc forward declarations */
+void OSMalloc_init(void);
+void OSMalloc_Tagref(OSMallocTag       tag);
+void OSMalloc_Tagrele(OSMallocTag      tag);
+

 /*
  *     Initialize the memory allocator.  This should be called only
  *     once on a system wide basis (i.e. first processor to get here
 /*
  *     Initialize the memory allocator.  This should be called only
  *     once on a system wide basis (i.e. first processor to get here


@@ -309,14 +339,31 @@ kalloc_init(
 {
        kern_return_t retval;
        vm_offset_t min;
 {
        kern_return_t retval;
        vm_offset_t min;

-       vm_size_t size;
+       vm_size_t size, kalloc_map_size;

        register int i;
 
        register int i;
 

+       /* 
+        * Scale the kalloc_map_size to physical memory size: stay below 
+        * 1/8th the total zone map size, or 128 MB (for a 32-bit kernel).
+        */
+       kalloc_map_size = (vm_size_t)(sane_size >> 5);
+#if !__LP64__
+       if (kalloc_map_size > KALLOC_MAP_SIZE_MAX)
+               kalloc_map_size = KALLOC_MAP_SIZE_MAX;
+#endif /* !__LP64__ */
+       if (kalloc_map_size < KALLOC_MAP_SIZE_MIN)
+               kalloc_map_size = KALLOC_MAP_SIZE_MIN;
+

        retval = kmem_suballoc(kernel_map, &min, kalloc_map_size,
        retval = kmem_suballoc(kernel_map, &min, kalloc_map_size,

-                              FALSE, TRUE, &kalloc_map);
+                              FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT,
+                              &kalloc_map);
+

        if (retval != KERN_SUCCESS)
                panic("kalloc_init: kmem_suballoc failed");
 
        if (retval != KERN_SUCCESS)
                panic("kalloc_init: kmem_suballoc failed");
 

+       kalloc_map_min = min;
+       kalloc_map_max = min + kalloc_map_size - 1;
+

        /*
         *      Ensure that zones up to size 8192 bytes exist.
         *      This is desirable because messages are allocated
        /*
         *      Ensure that zones up to size 8192 bytes exist.
         *      This is desirable because messages are allocated


@@ -328,295 +375,455 @@ kalloc_init(
        else
                kalloc_max = PAGE_SIZE;
        kalloc_max_prerounded = kalloc_max / 2 + 1;
        else
                kalloc_max = PAGE_SIZE;
        kalloc_max_prerounded = kalloc_max / 2 + 1;

+       /* size it to be more than 16 times kalloc_max (256k) for allocations from kernel map */
+       kalloc_kernmap_size = (kalloc_max * 16) + 1;
+       kalloc_largest_allocated = kalloc_kernmap_size;

 
        /*
         *      Allocate a zone for each size we are going to handle.
 
        /*
         *      Allocate a zone for each size we are going to handle.

-        *      We specify non-paged memory.
+        *      We specify non-paged memory.  Don't charge the caller
+        *      for the allocation, as we aren't sure how the memory
+        *      will be handled.

         */
         */

-       for (i = 0, size = 1; size < kalloc_max; i++, size <<= 1) {
-               if (size < KALLOC_MINSIZE) {
-                       k_zone[i] = 0;
-                       continue;
-               }
-               if (size == KALLOC_MINSIZE) {
-                       first_k_zone = i;
-               }
+       for (i = 0; (size = k_zone_size[i]) < kalloc_max; i++) {

                k_zone[i] = zinit(size, k_zone_max[i] * size, size,
                                  k_zone_name[i]);
                k_zone[i] = zinit(size, k_zone_max[i] * size, size,
                                  k_zone_name[i]);

+               zone_change(k_zone[i], Z_CALLERACCT, FALSE);
+       }
+
+       /*
+        * Build the Direct LookUp Table for small allocations
+        */
+       for (i = 0, size = 0; i <= N_K_ZDLUT; i++, size += KALLOC_MINALIGN) {
+               int zindex = 0;
+
+               while ((vm_size_t)k_zone_size[zindex] < size)
+                       zindex++;
+
+               if (i == N_K_ZDLUT) {
+                       k_zindex_start = zindex;
+                       break;
+               }
+               k_zone_dlut[i] = (int8_t)zindex;

        }
        }

+
+#ifdef KALLOC_DEBUG
+       printf("kalloc_init: k_zindex_start %d\n", k_zindex_start);
+
+       /*
+        * Do a quick synthesis to see how well/badly we can
+        * find-a-zone for a given size.
+        * Useful when debugging/tweaking the array of zone sizes.
+        * Cache misses probably more critical than compare-branches!
+        */
+       for (i = 0; i < (int)N_K_ZONE; i++) {
+               vm_size_t testsize = (vm_size_t)k_zone_size[i] - 1;
+               int compare = 0;
+               int zindex;
+
+               if (testsize < MAX_SIZE_ZDLUT) {
+                       compare += 1;   /* 'if' (T) */
+
+                       long dindex = INDEX_ZDLUT(testsize);
+                       zindex = (int)k_zone_dlut[dindex];
+
+               } else if (testsize < kalloc_max_prerounded) {
+
+                       compare += 2;   /* 'if' (F), 'if' (T) */
+
+                       zindex = k_zindex_start;
+                       while ((vm_size_t)k_zone_size[zindex] < testsize) {
+                               zindex++;
+                               compare++;      /* 'while' (T) */
+                       }
+                       compare++;      /* 'while' (F) */
+               } else
+                       break;  /* not zone-backed */
+
+               zone_t z = k_zone[zindex];
+               printf("kalloc_init: req size %4lu: %11s took %d compare%s\n",
+                   (unsigned long)testsize, z->zone_name, compare,
+                   compare == 1 ? "" : "s");
+       }
+#endif
+       kalloc_lck_grp = lck_grp_alloc_init("kalloc.large", LCK_GRP_ATTR_NULL);
+       lck_mtx_init(&kalloc_lock, kalloc_lck_grp, LCK_ATTR_NULL);
+       OSMalloc_init();
+#ifdef MUTEX_ZONE      
+       lck_mtx_zone = zinit(sizeof(struct _lck_mtx_), 1024*256, 4096, "lck_mtx");
+#endif 

 }
 
 }
 

-vm_offset_t
+/*
+ * Given an allocation size, return the kalloc zone it belongs to.
+ * Direct LookUp Table variant.
+ */
+static __inline zone_t
+get_zone_dlut(vm_size_t size)
+{
+       long dindex = INDEX_ZDLUT(size);
+       int zindex = (int)k_zone_dlut[dindex];
+       return (k_zone[zindex]);
+}
+
+/* As above, but linear search k_zone_size[] for the next zone that fits. */
+
+static __inline zone_t
+get_zone_search(vm_size_t size, int zindex)
+{
+       assert(size < kalloc_max_prerounded);
+
+       while ((vm_size_t)k_zone_size[zindex] < size)
+               zindex++;
+
+       assert((unsigned)zindex < N_K_ZONE &&
+           (vm_size_t)k_zone_size[zindex] < kalloc_max);
+
+       return (k_zone[zindex]);
+}
+
+void *

 kalloc_canblock(
                vm_size_t       size,
                boolean_t       canblock)
 {
 kalloc_canblock(
                vm_size_t       size,
                boolean_t       canblock)
 {

-       register int zindex;
-       register vm_size_t allocsize;
-
-       /*
-        * If size is too large for a zone, then use kmem_alloc.
-        * (We use kmem_alloc instead of kmem_alloc_wired so that
-        * krealloc can use kmem_realloc.)
-        */
-
-       if (size >= kalloc_max_prerounded) {
-               vm_offset_t addr;
+       zone_t z;
+
+       if (size < MAX_SIZE_ZDLUT)
+               z = get_zone_dlut(size);
+       else if (size < kalloc_max_prerounded)
+               z = get_zone_search(size, k_zindex_start);
+       else {
+               /*
+                * If size is too large for a zone, then use kmem_alloc.
+                * (We use kmem_alloc instead of kmem_alloc_kobject so that
+                * krealloc can use kmem_realloc.)
+                */
+               vm_map_t alloc_map;
+               void *addr;

 
                /* kmem_alloc could block so we return if noblock */
                if (!canblock) {
 
                /* kmem_alloc could block so we return if noblock */
                if (!canblock) {

-                 return(0);
+                       return(NULL);

                }
                }

-               if (kmem_alloc(kalloc_map, &addr, size) != KERN_SUCCESS)
-                       addr = 0;

 
 

-               if (addr) {
+               if (size >= kalloc_kernmap_size)
+                       alloc_map = kernel_map;
+               else
+                       alloc_map = kalloc_map;
+
+               if (kmem_alloc(alloc_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS) {
+                       if (alloc_map != kernel_map) {
+                               if (kmem_alloc(kernel_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS)
+                                       addr = NULL;
+                       }
+                       else
+                               addr = NULL;
+               }
+
+               if (addr != NULL) {
+                       kalloc_spin_lock();
+                       /*
+                        * Thread-safe version of the workaround for 4740071
+                        * (a double FREE())
+                        */
+                       if (size > kalloc_largest_allocated)
+                               kalloc_largest_allocated = size;
+

                        kalloc_large_inuse++;
                        kalloc_large_total += size;
                        kalloc_large_inuse++;
                        kalloc_large_total += size;

+                       kalloc_large_sum += size;

 
                        if (kalloc_large_total > kalloc_large_max)
                                kalloc_large_max = kalloc_large_total;
 
                        if (kalloc_large_total > kalloc_large_max)
                                kalloc_large_max = kalloc_large_total;

-               }
-               return(addr);
-       }

 
 

-       /* compute the size of the block that we will actually allocate */
+                       kalloc_unlock();

 
 

-       allocsize = KALLOC_MINSIZE;
-       zindex = first_k_zone;
-       while (allocsize < size) {
-               allocsize <<= 1;
-               zindex++;
+                       KALLOC_ZINFO_SALLOC(size);
+               }
+               return(addr);

        }
        }

-
-       /* allocate from the appropriate zone */
-
-       assert(allocsize < kalloc_max);
-       return(zalloc_canblock(k_zone[zindex], canblock));
+#ifdef KALLOC_DEBUG
+       if (size > z->elem_size)
+               panic("%s: z %p (%s) but requested size %lu", __func__,
+                   z, z->zone_name, (unsigned long)size);
+#endif
+       assert(size <= z->elem_size);
+       return (zalloc_canblock(z, canblock));

 }
 
 }
 

-vm_offset_t
+void *

 kalloc(
        vm_size_t size)
 {
 kalloc(
        vm_size_t size)
 {

-  return( kalloc_canblock(size, TRUE) );
+       return( kalloc_canblock(size, TRUE) );

 }
 
 }
 

-vm_offset_t
+void *

 kalloc_noblock(
               vm_size_t size)
 {
 kalloc_noblock(
               vm_size_t size)
 {

-  return( kalloc_canblock(size, FALSE) );
+       return( kalloc_canblock(size, FALSE) );

 }
 
 }
 

+volatile SInt32 kfree_nop_count = 0;

 
 void
 
 void

-krealloc(
-       vm_offset_t     *addrp,
-       vm_size_t       old_size,
-       vm_size_t       new_size,
-       simple_lock_t   lock)
+kfree(
+       void            *data,
+       vm_size_t       size)

 {
 {

-       register int zindex;
-       register vm_size_t allocsize;
-       vm_offset_t naddr;
+       zone_t z;
+
+       if (size < MAX_SIZE_ZDLUT)
+               z = get_zone_dlut(size);
+       else if (size < kalloc_max_prerounded)
+               z = get_zone_search(size, k_zindex_start);
+       else {
+               /* if size was too large for a zone, then use kmem_free */
+
+               vm_map_t alloc_map = kernel_map;
+
+               if ((((vm_offset_t) data) >= kalloc_map_min) && (((vm_offset_t) data) <= kalloc_map_max))
+                       alloc_map = kalloc_map;
+               if (size > kalloc_largest_allocated) {
+                               /*
+                                * work around double FREEs of small MALLOCs
+                                * this used to end up being a nop
+                                * since the pointer being freed from an
+                                * alloc backed by the zalloc world could
+                                * never show up in the kalloc_map... however,
+                                * the kernel_map is a different issue... since it
+                                * was released back into the zalloc pool, a pointer
+                                * would have gotten written over the 'size' that 
+                                * the MALLOC was retaining in the first 4 bytes of
+                                * the underlying allocation... that pointer ends up 
+                                * looking like a really big size on the 2nd FREE and
+                                * pushes the kfree into the kernel_map...  we
+                                * end up removing a ton of virtual space before we panic
+                                * this check causes us to ignore the kfree for a size
+                                * that must be 'bogus'... note that it might not be due
+                                * to the above scenario, but it would still be wrong and
+                                * cause serious damage.
+                                */
+
+                               OSAddAtomic(1, &kfree_nop_count);
+                               return;
+               }
+               kmem_free(alloc_map, (vm_offset_t)data, size);

 
 

-       /* can only be used for increasing allocation size */
+               kalloc_spin_lock();

 
 

-       assert(new_size > old_size);
+               kalloc_large_total -= size;
+               kalloc_large_inuse--;

 
 

-       /* if old_size is zero, then we are simply allocating */
+               kalloc_unlock();

 
 

-       if (old_size == 0) {
-               simple_unlock(lock);
-               naddr = kalloc(new_size);
-               simple_lock(lock);
-               *addrp = naddr;
+               KALLOC_ZINFO_SFREE(size);

                return;
        }
 
                return;
        }
 

-       /* if old block was kmem_alloc'd, then use kmem_realloc if necessary */
-
-       if (old_size >= kalloc_max_prerounded) {
-               old_size = round_page(old_size);
-               new_size = round_page(new_size);
-               if (new_size > old_size) {
-
-                       if (kmem_realloc(kalloc_map, *addrp, old_size, &naddr,
-                                        new_size) != KERN_SUCCESS) {
-                               panic("krealloc: kmem_realloc");
-                               naddr = 0;
-                       }
-
-                       simple_lock(lock);
-                       *addrp = naddr;
-
-                       /* kmem_realloc() doesn't free old page range. */
-                       kmem_free(kalloc_map, *addrp, old_size);
-
-                       kalloc_large_total += (new_size - old_size);
-
-                       if (kalloc_large_total > kalloc_large_max)
-                               kalloc_large_max = kalloc_large_total;
-               }
-               return;
-       }
+       /* free to the appropriate zone */
+#ifdef KALLOC_DEBUG
+       if (size > z->elem_size)
+               panic("%s: z %p (%s) but requested size %lu", __func__,
+                   z, z->zone_name, (unsigned long)size);
+#endif
+       assert(size <= z->elem_size);
+       zfree(z, data);
+}

 
 

-       /* compute the size of the block that we actually allocated */
+#ifdef MACH_BSD
+zone_t
+kalloc_zone(
+       vm_size_t       size)
+{
+       if (size < MAX_SIZE_ZDLUT)
+               return (get_zone_dlut(size));
+       if (size <= kalloc_max)
+               return (get_zone_search(size, k_zindex_start));
+       return (ZONE_NULL);
+}
+#endif

 
 

-       allocsize = KALLOC_MINSIZE;
-       zindex = first_k_zone;
-       while (allocsize < old_size) {
-               allocsize <<= 1;
-               zindex++;
-       }
+void
+kalloc_fake_zone_init(int zone_index)
+{
+       kalloc_fake_zone_index = zone_index;
+}

 
 

-       /* if new size fits in old block, then return */
+void
+kalloc_fake_zone_info(int *count, 
+                     vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size,
+                     uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct)
+{
+       *count      = kalloc_large_inuse;
+       *cur_size   = kalloc_large_total;
+       *max_size   = kalloc_large_max;

 
 

-       if (new_size <= allocsize) {
-               return;
+       if (kalloc_large_inuse) {
+               *elem_size  = kalloc_large_total / kalloc_large_inuse;
+               *alloc_size = kalloc_large_total / kalloc_large_inuse;
+       } else {
+               *elem_size  = 0;
+               *alloc_size = 0;

        }
        }

+       *sum_size   = kalloc_large_sum;
+       *collectable = 0;
+       *exhaustable = 0;
+       *caller_acct = 0;
+}

 
 

-       /* if new size does not fit in zone, kmem_alloc it, else zalloc it */

 
 

-       simple_unlock(lock);
-       if (new_size >= kalloc_max_prerounded) {
-               if (kmem_alloc(kalloc_map, &naddr, new_size) != KERN_SUCCESS) {
-                       panic("krealloc: kmem_alloc");
-                       simple_lock(lock);
-                       *addrp = 0;
-                       return;
-               }
-               kalloc_large_inuse++;
-               kalloc_large_total += new_size;
+void
+OSMalloc_init(
+       void)
+{
+       queue_init(&OSMalloc_tag_list);

 
 

-               if (kalloc_large_total > kalloc_large_max)
-                       kalloc_large_max = kalloc_large_total;
-       } else {
-               register int new_zindex;
+       OSMalloc_tag_lck_grp = lck_grp_alloc_init("OSMalloc_tag", LCK_GRP_ATTR_NULL);
+       lck_mtx_init(&OSMalloc_tag_lock, OSMalloc_tag_lck_grp, LCK_ATTR_NULL);
+}

 
 

-               allocsize <<= 1;
-               new_zindex = zindex + 1;
-               while (allocsize < new_size) {
-                       allocsize <<= 1;
-                       new_zindex++;
-               }
-               naddr = zalloc(k_zone[new_zindex]);
-       }
-       simple_lock(lock);
+OSMallocTag
+OSMalloc_Tagalloc(
+       const char                      *str,
+       uint32_t                        flags)
+{
+       OSMallocTag       OSMTag;

 
 

-       /* copy existing data */
+       OSMTag = (OSMallocTag)kalloc(sizeof(*OSMTag));

 
 

-       bcopy((const char *)*addrp, (char *)naddr, old_size);
+       bzero((void *)OSMTag, sizeof(*OSMTag));

 
 

-       /* free old block, and return */
+       if (flags & OSMT_PAGEABLE)
+               OSMTag->OSMT_attr = OSMT_ATTR_PAGEABLE;

 
 

-       zfree(k_zone[zindex], *addrp);
+       OSMTag->OSMT_refcnt = 1;

 
 

-       /* set up new address */
+       strncpy(OSMTag->OSMT_name, str, OSMT_MAX_NAME);

 
 

-       *addrp = naddr;
+       OSMalloc_tag_spin_lock();
+       enqueue_tail(&OSMalloc_tag_list, (queue_entry_t)OSMTag);
+       OSMalloc_tag_unlock();
+       OSMTag->OSMT_state = OSMT_VALID;
+       return(OSMTag);

 }
 
 }
 

-
-vm_offset_t
-kget(
-       vm_size_t       size)
+void
+OSMalloc_Tagref(
+        OSMallocTag            tag)

 {
 {

-       register int zindex;
-       register vm_size_t allocsize;
+       if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID)) 
+               panic("OSMalloc_Tagref():'%s' has bad state 0x%08X\n", tag->OSMT_name, tag->OSMT_state);

 
 

-       /* size must not be too large for a zone */
+       (void)hw_atomic_add(&tag->OSMT_refcnt, 1);
+}

 
 

-       if (size >= kalloc_max_prerounded) {
-               /* This will never work, so we might as well panic */
-               panic("kget");
+void
+OSMalloc_Tagrele(
+        OSMallocTag            tag)
+{
+       if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID))
+               panic("OSMalloc_Tagref():'%s' has bad state 0x%08X\n", tag->OSMT_name, tag->OSMT_state);
+
+       if (hw_atomic_sub(&tag->OSMT_refcnt, 1) == 0) {
+               if (hw_compare_and_store(OSMT_VALID|OSMT_RELEASED, OSMT_VALID|OSMT_RELEASED, &tag->OSMT_state)) {
+                       OSMalloc_tag_spin_lock();
+                       (void)remque((queue_entry_t)tag);
+                       OSMalloc_tag_unlock();
+                       kfree((void*)tag, sizeof(*tag));
+               } else
+                       panic("OSMalloc_Tagrele():'%s' has refcnt 0\n", tag->OSMT_name);

        }
        }

+}

 
 

-       /* compute the size of the block that we will actually allocate */
-
-       allocsize = KALLOC_MINSIZE;
-       zindex = first_k_zone;
-       while (allocsize < size) {
-               allocsize <<= 1;
-               zindex++;
+void
+OSMalloc_Tagfree(
+        OSMallocTag            tag)
+{
+       if (!hw_compare_and_store(OSMT_VALID, OSMT_VALID|OSMT_RELEASED, &tag->OSMT_state))
+               panic("OSMalloc_Tagfree():'%s' has bad state 0x%08X \n", tag->OSMT_name, tag->OSMT_state);
+
+       if (hw_atomic_sub(&tag->OSMT_refcnt, 1) == 0) {
+               OSMalloc_tag_spin_lock();
+               (void)remque((queue_entry_t)tag);
+               OSMalloc_tag_unlock();
+               kfree((void*)tag, sizeof(*tag));

        }
        }

-
-       /* allocate from the appropriate zone */
-
-       assert(allocsize < kalloc_max);
-       return(zget(k_zone[zindex]));

 }
 
 }
 

-void
-kfree(
-       vm_offset_t     data,
-       vm_size_t       size)
+void *
+OSMalloc(
+       uint32_t                        size,
+       OSMallocTag                     tag)

 {
 {

-       register int zindex;
-       register vm_size_t freesize;
+       void                    *addr=NULL;
+       kern_return_t   kr;

 
 

-       /* if size was too large for a zone, then use kmem_free */
+       OSMalloc_Tagref(tag);
+       if ((tag->OSMT_attr & OSMT_PAGEABLE)
+           && (size & ~PAGE_MASK)) {

 
 

-       if (size >= kalloc_max_prerounded) {
-               kmem_free(kalloc_map, data, size);
+               if ((kr = kmem_alloc_pageable(kernel_map, (vm_offset_t *)&addr, size)) != KERN_SUCCESS)
+                       addr = NULL;
+       } else 
+               addr = kalloc((vm_size_t)size);

 
 

-               kalloc_large_total -= size;
-               kalloc_large_inuse--;
+       if (!addr)
+               OSMalloc_Tagrele(tag);

 
 

-               return;
-       }
+       return(addr);
+}

 
 

-       /* compute the size of the block that we actually allocated from */
+void *
+OSMalloc_nowait(
+       uint32_t                        size,
+       OSMallocTag                     tag)
+{
+       void    *addr=NULL;

 
 

-       freesize = KALLOC_MINSIZE;
-       zindex = first_k_zone;
-       while (freesize < size) {
-               freesize <<= 1;
-               zindex++;
-       }
+       if (tag->OSMT_attr & OSMT_PAGEABLE)
+               return(NULL);

 
 

-       /* free to the appropriate zone */
+       OSMalloc_Tagref(tag);
+       /* XXX: use non-blocking kalloc for now */
+       addr = kalloc_noblock((vm_size_t)size);
+       if (addr == NULL)
+               OSMalloc_Tagrele(tag);

 
 

-       assert(freesize < kalloc_max);
-       zfree(k_zone[zindex], data);
+       return(addr);

 }
 
 }
 

-#ifdef MACH_BSD
-zone_t
-kalloc_zone(
-       vm_size_t       size)
+void *
+OSMalloc_noblock(
+       uint32_t                        size,
+       OSMallocTag                     tag)

 {
 {

-       register int zindex = 0;
-       register vm_size_t allocsize;
+       void    *addr=NULL;

 
 

-       /* compute the size of the block that we will actually allocate */
-
-       allocsize = size;
-       if (size <= kalloc_max) {
-               allocsize = KALLOC_MINSIZE;
-               zindex = first_k_zone;
-               while (allocsize < size) {
-                       allocsize <<= 1;
-                       zindex++;
-               }
-               return (k_zone[zindex]);
-       }
-       return (ZONE_NULL);
-}
-#endif
+       if (tag->OSMT_attr & OSMT_PAGEABLE)
+               return(NULL);

 
 

+       OSMalloc_Tagref(tag);
+       addr = kalloc_noblock((vm_size_t)size);
+       if (addr == NULL)
+               OSMalloc_Tagrele(tag);

 
 

+       return(addr);
+}

 
 

-kalloc_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size,
-                    vm_size_t *alloc_size, int *collectable, int *exhaustable)
+void
+OSFree(
+       void                            *addr,
+       uint32_t                        size,
+       OSMallocTag                     tag) 

 {
 {

-        *count      = kalloc_large_inuse;
-       *cur_size   = kalloc_large_total;
-       *max_size   = kalloc_large_max;
-       *elem_size  = kalloc_large_total / kalloc_large_inuse;
-       *alloc_size = kalloc_large_total / kalloc_large_inuse;
-       *collectable = 0;
-       *exhaustable = 0;
-}
+       if ((tag->OSMT_attr & OSMT_PAGEABLE)
+           && (size & ~PAGE_MASK)) {
+               kmem_free(kernel_map, (vm_offset_t)addr, size);
+       } else
+               kfree((void *)addr, size);

 
 

+       OSMalloc_Tagrele(tag);
+}