]>
git.saurik.com Git - apple/xnu.git/blob - bsd/dev/dtrace/blist.c
2 * BLIST.C - Bitmap allocator/deallocator, using a radix tree with hinting
4 * (c)Copyright 1998, Matthew Dillon. Terms for use and redistribution
5 * are covered by the BSD Copyright as found in /usr/src/COPYRIGHT.
7 * This module implements a general bitmap allocator/deallocator. The
8 * allocator eats around 2 bits per 'block'. The module does not
9 * try to interpret the meaning of a 'block' other then to return
10 * SWAPBLK_NONE on an allocation failure.
12 * A radix tree is used to maintain the bitmap. Two radix constants are
13 * involved: One for the bitmaps contained in the leaf nodes (typically
14 * 32), and one for the meta nodes (typically 16). Both meta and leaf
15 * nodes have a hint field. This field gives us a hint as to the largest
16 * free contiguous range of blocks under the node. It may contain a
17 * value that is too high, but will never contain a value that is too
18 * low. When the radix tree is searched, allocation failures in subtrees
21 * The radix tree also implements two collapsed states for meta nodes:
22 * the ALL-ALLOCATED state and the ALL-FREE state. If a meta node is
23 * in either of these two states, all information contained underneath
24 * the node is considered stale. These states are used to optimize
25 * allocation and freeing operations.
27 * The hinting greatly increases code efficiency for allocations while
28 * the general radix structure optimizes both allocations and frees. The
29 * radix tree should be able to operate well no matter how much
30 * fragmentation there is and no matter how large a bitmap is used.
32 * Unlike the rlist code, the blist code wires all necessary memory at
33 * creation time. Neither allocations nor frees require interaction with
34 * the memory subsystem. In contrast, the rlist code may allocate memory
35 * on an rlist_free() call. The non-blocking features of the blist code
36 * are used to great advantage in the swap code (vm/nswap_pager.c). The
37 * rlist code uses a little less overall memory then the blist code (but
38 * due to swap interleaving not all that much less), but the blist code
39 * scales much, much better.
41 * LAYOUT: The radix tree is layed out recursively using a
42 * linear array. Each meta node is immediately followed (layed out
43 * sequentially in memory) by BLIST_META_RADIX lower level nodes. This
44 * is a recursive structure but one that can be easily scanned through
45 * a very simple 'skip' calculation. In order to support large radixes,
46 * portions of the tree may reside outside our memory allocation. We
47 * handle this with an early-termination optimization (when bighint is
48 * set to -1) on the scan. The memory allocation is only large enough
49 * to cover the number of blocks requested at creation time even if it
50 * must be encompassed in larger root-node radix.
52 * NOTE: the allocator cannot currently allocate more then
53 * BLIST_BMAP_RADIX blocks per call. It will panic with 'allocation too
54 * large' if you try. This is an area that could use improvement. The
55 * radix is large enough that this restriction does not effect the swap
56 * system, though. Currently only the allocation code is effected by
57 * this algorithmic unfeature. The freeing code can handle arbitrary
60 * This code can be compiled stand-alone for debugging.
62 * $FreeBSD: src/sys/kern/subr_blist.c,v 1.5.2.1 2000/03/17 10:47:29 ps Exp $
65 #if !defined(__APPLE__)
68 #include <sys/param.h>
69 #include <sys/systm.h>
71 #include <sys/kernel.h>
72 #include <sys/blist.h>
73 #include <sys/malloc.h>
75 #include <vm/vm_object.h>
76 #include <vm/vm_kern.h>
77 #include <vm/vm_extern.h>
78 #include <vm/vm_page.h>
82 #ifndef BLIST_NO_DEBUG
86 #define SWAPBLK_NONE ((daddr_t)-1)
88 #include <sys/types.h>
94 #define malloc(a, b, c) malloc(a)
95 #define free(a, b) free(a)
97 typedef unsigned int u_daddr_t
;
99 #include <sys/blist.h>
101 void panic(const char *ctl
, ...);
104 #else /* is MacOS X */
107 #define _KERNEL /* Solaris vs. Darwin */
111 typedef unsigned int u_daddr_t
;
113 #include <sys/param.h>
114 #include <sys/systm.h>
115 #include <sys/lock.h>
116 #include <sys/kernel.h>
117 /* #include <sys/blist.h> */
119 #include <sys/malloc.h>
121 #define SWAPBLK_NONE ((daddr_t)-1)
122 #define malloc _MALLOC
124 #define M_SWAP M_TEMP
126 #endif /* __APPLE__ */
129 * static support functions
132 static daddr_t
blst_leaf_alloc(blmeta_t
*scan
, daddr_t blk
, int count
);
133 static daddr_t
blst_meta_alloc(blmeta_t
*scan
, daddr_t blk
,
134 daddr_t count
, daddr_t radix
, int skip
);
135 static void blst_leaf_free(blmeta_t
*scan
, daddr_t relblk
, int count
);
136 static void blst_meta_free(blmeta_t
*scan
, daddr_t freeBlk
, daddr_t count
,
137 daddr_t radix
, int skip
, daddr_t blk
);
138 static void blst_copy(blmeta_t
*scan
, daddr_t blk
, daddr_t radix
,
139 daddr_t skip
, blist_t dest
, daddr_t count
);
140 static daddr_t
blst_radix_init(blmeta_t
*scan
, daddr_t radix
,
141 int skip
, daddr_t count
);
143 static void blst_radix_print(blmeta_t
*scan
, daddr_t blk
,
144 daddr_t radix
, int skip
, int tab
);
147 #if !defined(__APPLE__)
149 static MALLOC_DEFINE(M_SWAP
, "SWAP", "Swap space");
151 #endif /* __APPLE__ */
154 * blist_create() - create a blist capable of handling up to the specified
157 * blocks must be greater then 0
159 * The smallest blist consists of a single leaf node capable of
160 * managing BLIST_BMAP_RADIX blocks.
164 blist_create(daddr_t blocks
)
171 * Calculate radix and skip field used for scanning.
173 radix
= BLIST_BMAP_RADIX
;
175 while (radix
< blocks
) {
176 radix
<<= BLIST_META_RADIX_SHIFT
;
177 skip
= (skip
+ 1) << BLIST_META_RADIX_SHIFT
;
180 bl
= malloc(sizeof(struct blist
), M_SWAP
, M_WAITOK
);
182 bzero(bl
, sizeof(*bl
));
184 bl
->bl_blocks
= blocks
;
185 bl
->bl_radix
= radix
;
187 bl
->bl_rootblks
= 1 +
188 blst_radix_init(NULL
, bl
->bl_radix
, bl
->bl_skip
, blocks
);
189 bl
->bl_root
= malloc(sizeof(blmeta_t
) * bl
->bl_rootblks
, M_SWAP
, M_WAITOK
);
191 #if defined(BLIST_DEBUG)
193 "BLIST representing %d blocks (%d MB of swap)"
194 ", requiring %dK of ram\n",
196 bl
->bl_blocks
* 4 / 1024,
197 (bl
->bl_rootblks
* sizeof(blmeta_t
) + 1023) / 1024
199 printf("BLIST raw radix tree contains %d records\n", bl
->bl_rootblks
);
201 blst_radix_init(bl
->bl_root
, bl
->bl_radix
, bl
->bl_skip
, blocks
);
207 blist_destroy(blist_t bl
)
209 free(bl
->bl_root
, M_SWAP
);
214 * blist_alloc() - reserve space in the block bitmap. Return the base
215 * of a contiguous region or SWAPBLK_NONE if space could
220 blist_alloc(blist_t bl
, daddr_t count
)
222 daddr_t blk
= SWAPBLK_NONE
;
225 if (bl
->bl_radix
== BLIST_BMAP_RADIX
) {
226 blk
= blst_leaf_alloc(bl
->bl_root
, 0, count
);
228 blk
= blst_meta_alloc(bl
->bl_root
, 0, count
,
229 bl
->bl_radix
, bl
->bl_skip
);
231 if (blk
!= SWAPBLK_NONE
) {
232 bl
->bl_free
-= count
;
239 * blist_free() - free up space in the block bitmap. Return the base
240 * of a contiguous region. Panic if an inconsistancy is
245 blist_free(blist_t bl
, daddr_t blkno
, daddr_t count
)
248 if (bl
->bl_radix
== BLIST_BMAP_RADIX
) {
249 blst_leaf_free(bl
->bl_root
, blkno
, count
);
251 blst_meta_free(bl
->bl_root
, blkno
, count
,
252 bl
->bl_radix
, bl
->bl_skip
, 0);
254 bl
->bl_free
+= count
;
259 * blist_resize() - resize an existing radix tree to handle the
260 * specified number of blocks. This will reallocate
261 * the tree and transfer the previous bitmap to the new
262 * one. When extending the tree you can specify whether
263 * the new blocks are to left allocated or freed.
267 blist_resize(blist_t
*pbl
, daddr_t count
, int freenew
)
269 blist_t newbl
= blist_create(count
);
273 if (count
> save
->bl_blocks
) {
274 count
= save
->bl_blocks
;
276 blst_copy(save
->bl_root
, 0, save
->bl_radix
, save
->bl_skip
, newbl
, count
);
279 * If resizing upwards, should we free the new space or not?
281 if (freenew
&& count
< newbl
->bl_blocks
) {
282 blist_free(newbl
, count
, newbl
->bl_blocks
- count
);
290 * blist_print() - dump radix tree
294 blist_print(blist_t bl
)
297 blst_radix_print(bl
->bl_root
, 0, bl
->bl_radix
, bl
->bl_skip
, 4);
303 /************************************************************************
304 * ALLOCATION SUPPORT FUNCTIONS *
305 ************************************************************************
307 * These support functions do all the actual work. They may seem
308 * rather longish, but that's because I've commented them up. The
309 * actual code is straight forward.
314 * blist_leaf_alloc() - allocate at a leaf in the radix tree (a bitmap).
316 * This is the core of the allocator and is optimized for the 1 block
317 * and the BLIST_BMAP_RADIX block allocation cases. Other cases are
318 * somewhat slower. The 1 block allocation case is log2 and extremely
323 blst_leaf_alloc(blmeta_t
*scan
, daddr_t blk
, int count
)
325 u_daddr_t orig
= scan
->u
.bmu_bitmap
;
329 * Optimize bitmap all-allocated case. Also, count = 1
330 * case assumes at least 1 bit is free in the bitmap, so
331 * we have to take care of this case here.
333 scan
->bm_bighint
= 0;
338 * Optimized code to allocate one bit out of the bitmap
341 int j
= BLIST_BMAP_RADIX
/ 2;
344 mask
= (u_daddr_t
)-1 >> (BLIST_BMAP_RADIX
/ 2);
347 if ((orig
& mask
) == 0) {
354 scan
->u
.bmu_bitmap
&= ~(1 << r
);
357 #if !defined(__APPLE__)
358 if (count
<= BLIST_BMAP_RADIX
) {
360 if (count
<= (int)BLIST_BMAP_RADIX
) {
361 #endif /* __APPLE__ */
363 * non-optimized code to allocate N bits out of the bitmap.
364 * The more bits, the faster the code runs. It will run
365 * the slowest allocating 2 bits, but since there aren't any
366 * memory ops in the core loop (or shouldn't be, anyway),
367 * you probably won't notice the difference.
370 int n
= BLIST_BMAP_RADIX
- count
;
373 mask
= (u_daddr_t
)-1 >> n
;
375 for (j
= 0; j
<= n
; ++j
) {
376 if ((orig
& mask
) == mask
) {
377 scan
->u
.bmu_bitmap
&= ~mask
;
384 * We couldn't allocate count in this subtree, update bighint.
386 scan
->bm_bighint
= count
- 1;
391 * blist_meta_alloc() - allocate at a meta in the radix tree.
393 * Attempt to allocate at a meta node. If we can't, we update
394 * bighint and return a failure. Updating bighint optimize future
395 * calls that hit this node. We have to check for our collapse cases
396 * and we have a few optimizations strewn in as well.
400 blst_meta_alloc(blmeta_t
*scan
, daddr_t blk
, daddr_t count
, daddr_t radix
,
404 int next_skip
= (skip
>> BLIST_META_RADIX_SHIFT
);
406 if (scan
->u
.bmu_avail
== 0) {
408 * ALL-ALLOCATED special case
410 scan
->bm_bighint
= count
;
414 if (scan
->u
.bmu_avail
== radix
) {
415 radix
>>= BLIST_META_RADIX_SHIFT
;
418 * ALL-FREE special case, initialize uninitialize
421 for (i
= 1; i
<= skip
; i
+= next_skip
) {
422 if (scan
[i
].bm_bighint
== (daddr_t
)-1) {
425 if (next_skip
== 1) {
426 scan
[i
].u
.bmu_bitmap
= (u_daddr_t
)-1;
427 scan
[i
].bm_bighint
= BLIST_BMAP_RADIX
;
429 scan
[i
].bm_bighint
= radix
;
430 scan
[i
].u
.bmu_avail
= radix
;
434 radix
>>= BLIST_META_RADIX_SHIFT
;
437 for (i
= 1; i
<= skip
; i
+= next_skip
) {
438 if (count
<= scan
[i
].bm_bighint
) {
440 * count fits in object
443 if (next_skip
== 1) {
444 r
= blst_leaf_alloc(&scan
[i
], blk
, count
);
446 r
= blst_meta_alloc(&scan
[i
], blk
, count
,
447 radix
, next_skip
- 1);
449 if (r
!= SWAPBLK_NONE
) {
450 scan
->u
.bmu_avail
-= count
;
451 if (scan
->bm_bighint
> scan
->u
.bmu_avail
) {
452 scan
->bm_bighint
= scan
->u
.bmu_avail
;
456 } else if (scan
[i
].bm_bighint
== (daddr_t
)-1) {
461 } else if (count
> radix
) {
463 * count does not fit in object even if it were
466 panic("blist_meta_alloc: allocation too large");
472 * We couldn't allocate count in this subtree, update bighint.
474 if (scan
->bm_bighint
>= count
) {
475 scan
->bm_bighint
= count
- 1;
481 * BLST_LEAF_FREE() - free allocated block from leaf bitmap
486 blst_leaf_free(blmeta_t
*scan
, daddr_t blk
, int count
)
489 * free some data in this bitmap
492 * 0000111111111110000
496 int n
= blk
& (BLIST_BMAP_RADIX
- 1);
499 mask
= ((u_daddr_t
)-1 << n
) &
500 ((u_daddr_t
)-1 >> (BLIST_BMAP_RADIX
- count
- n
));
502 if (scan
->u
.bmu_bitmap
& mask
) {
503 panic("blst_radix_free: freeing free block");
505 scan
->u
.bmu_bitmap
|= mask
;
508 * We could probably do a better job here. We are required to make
509 * bighint at least as large as the biggest contiguous block of
510 * data. If we just shoehorn it, a little extra overhead will
511 * be incured on the next allocation (but only that one typically).
513 scan
->bm_bighint
= BLIST_BMAP_RADIX
;
517 * BLST_META_FREE() - free allocated blocks from radix tree meta info
519 * This support routine frees a range of blocks from the bitmap.
520 * The range must be entirely enclosed by this radix node. If a
521 * meta node, we break the range down recursively to free blocks
522 * in subnodes (which means that this code can free an arbitrary
523 * range whereas the allocation code cannot allocate an arbitrary
528 blst_meta_free(blmeta_t
*scan
, daddr_t freeBlk
, daddr_t count
, daddr_t radix
,
529 int skip
, daddr_t blk
)
532 int next_skip
= (skip
>> BLIST_META_RADIX_SHIFT
);
535 printf("FREE (%x,%d) FROM (%x,%d)\n",
541 if (scan
->u
.bmu_avail
== 0) {
543 * ALL-ALLOCATED special case, with possible
544 * shortcut to ALL-FREE special case.
546 scan
->u
.bmu_avail
= count
;
547 scan
->bm_bighint
= count
;
549 if (count
!= radix
) {
550 for (i
= 1; i
<= skip
; i
+= next_skip
) {
551 if (scan
[i
].bm_bighint
== (daddr_t
)-1) {
554 scan
[i
].bm_bighint
= 0;
555 if (next_skip
== 1) {
556 scan
[i
].u
.bmu_bitmap
= 0;
558 scan
[i
].u
.bmu_avail
= 0;
564 scan
->u
.bmu_avail
+= count
;
565 /* scan->bm_bighint = radix; */
569 * ALL-FREE special case.
572 if (scan
->u
.bmu_avail
== radix
) {
575 if (scan
->u
.bmu_avail
> radix
) {
576 panic("blst_meta_free: freeing already free blocks (%d) %d/%d", count
, scan
->u
.bmu_avail
, radix
);
580 * Break the free down into its components
583 radix
>>= BLIST_META_RADIX_SHIFT
;
585 i
= (freeBlk
- blk
) / radix
;
587 i
= i
* next_skip
+ 1;
589 while (i
<= skip
&& blk
< freeBlk
+ count
) {
592 v
= blk
+ radix
- freeBlk
;
597 if (scan
->bm_bighint
== (daddr_t
)-1) {
598 panic("blst_meta_free: freeing unexpected range");
601 if (next_skip
== 1) {
602 blst_leaf_free(&scan
[i
], freeBlk
, v
);
604 blst_meta_free(&scan
[i
], freeBlk
, v
, radix
,
607 if (scan
->bm_bighint
< scan
[i
].bm_bighint
) {
608 scan
->bm_bighint
= scan
[i
].bm_bighint
;
618 * BLIST_RADIX_COPY() - copy one radix tree to another
620 * Locates free space in the source tree and frees it in the destination
621 * tree. The space may not already be free in the destination.
625 blst_copy(blmeta_t
*scan
, daddr_t blk
, daddr_t radix
,
626 daddr_t skip
, blist_t dest
, daddr_t count
)
635 if (radix
== BLIST_BMAP_RADIX
) {
636 u_daddr_t v
= scan
->u
.bmu_bitmap
;
638 if (v
== (u_daddr_t
)-1) {
639 blist_free(dest
, blk
, count
);
641 #if !defined(__APPLE__)
644 for (i
= 0; i
< BLIST_BMAP_RADIX
&& i
< count
; ++i
) {
646 blist_free(dest
, blk
+ i
, 1);
650 int j
; /* Avoid shadow warnings */
652 for (j
= 0; j
< (int)BLIST_BMAP_RADIX
&& j
< count
; ++j
) {
654 blist_free(dest
, blk
+ j
, 1);
657 #endif /* __APPLE__ */
667 * Source all allocated, leave dest allocated
669 if (scan
->u
.bmu_avail
== 0) {
672 if (scan
->u
.bmu_avail
== radix
) {
674 * Source all free, free entire dest
677 blist_free(dest
, blk
, count
);
679 blist_free(dest
, blk
, radix
);
684 radix
>>= BLIST_META_RADIX_SHIFT
;
685 next_skip
= (skip
>> BLIST_META_RADIX_SHIFT
);
687 for (i
= 1; count
&& i
<= skip
; i
+= next_skip
) {
688 if (scan
[i
].bm_bighint
== (daddr_t
)-1) {
692 if (count
>= radix
) {
720 * BLST_RADIX_INIT() - initialize radix tree
722 * Initialize our meta structures and bitmaps and calculate the exact
723 * amount of space required to manage 'count' blocks - this space may
724 * be considerably less then the calculated radix due to the large
725 * RADIX values we use.
729 blst_radix_init(blmeta_t
*scan
, daddr_t radix
, int skip
, daddr_t count
)
733 daddr_t memindex
= 0;
739 if (radix
== BLIST_BMAP_RADIX
) {
741 scan
->bm_bighint
= 0;
742 scan
->u
.bmu_bitmap
= 0;
748 * Meta node. If allocating the entire object we can special
749 * case it. However, we need to figure out how much memory
750 * is required to manage 'count' blocks, so we continue on anyway.
754 scan
->bm_bighint
= 0;
755 scan
->u
.bmu_avail
= 0;
758 radix
>>= BLIST_META_RADIX_SHIFT
;
759 next_skip
= (skip
>> BLIST_META_RADIX_SHIFT
);
761 for (i
= 1; i
<= skip
; i
+= next_skip
) {
762 if (count
>= radix
) {
764 * Allocate the entire object
766 memindex
= i
+ blst_radix_init(
767 ((scan
) ? &scan
[i
] : NULL
),
773 } else if (count
> 0) {
775 * Allocate a partial object
777 memindex
= i
+ blst_radix_init(
778 ((scan
) ? &scan
[i
] : NULL
),
786 * Add terminator and break out
789 scan
[i
].bm_bighint
= (daddr_t
)-1;
803 blst_radix_print(blmeta_t
*scan
, daddr_t blk
, daddr_t radix
, int skip
, int tab
)
809 if (radix
== BLIST_BMAP_RADIX
) {
811 "%*.*s(%04x,%d): bitmap %08x big=%d\n",
820 if (scan
->u
.bmu_avail
== 0) {
822 "%*.*s(%04x,%d) ALL ALLOCATED\n",
829 if (scan
->u
.bmu_avail
== radix
) {
831 "%*.*s(%04x,%d) ALL FREE\n",
840 "%*.*s(%04x,%d): subtree (%d/%d) big=%d {\n",
848 radix
>>= BLIST_META_RADIX_SHIFT
;
849 next_skip
= (skip
>> BLIST_META_RADIX_SHIFT
);
852 for (i
= 1; i
<= skip
; i
+= next_skip
) {
853 if (scan
[i
].bm_bighint
== (daddr_t
)-1) {
855 "%*.*s(%04x,%d): Terminator\n",
884 main(int ac
, char **av
)
890 for (i
= 1; i
< ac
; ++i
) {
891 const char *ptr
= av
[i
];
893 size
= strtol(ptr
, NULL
, 0);
897 fprintf(stderr
, "Bad option: %s\n", ptr
- 2);
900 bl
= blist_create(size
);
901 blist_free(bl
, 0, size
);
909 printf("%d/%d/%d> ", bl
->bl_free
, size
, bl
->bl_radix
);
911 if (fgets(buf
, sizeof(buf
), stdin
) == NULL
) {
916 if (sscanf(buf
+ 1, "%d", &count
) == 1) {
917 blist_resize(&bl
, count
, 1);
925 if (sscanf(buf
+ 1, "%d", &count
) == 1) {
926 daddr_t blk
= blist_alloc(bl
, count
);
927 printf(" R=%04x\n", blk
);
933 if (sscanf(buf
+ 1, "%x %d", &da
, &count
) == 2) {
934 blist_free(bl
, da
, count
);
958 panic(const char *ctl
, ...)
963 vfprintf(stderr
, ctl
, va
);
964 fprintf(stderr
, "\n");