]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/zalloc.c
projects / apple / xnu.git / blob
? search:


fa4ad972e8707f735ffd026860b26b332017f5b4
[apple/xnu.git] / osfmk / kern / zalloc.c
1 /*
2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23 /*
24 * @OSF_COPYRIGHT@
25 */
26 /*
27 * Mach Operating System
28 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
29 * All Rights Reserved.
30 *
31 * Permission to use, copy, modify and distribute this software and its
32 * documentation is hereby granted, provided that both the copyright
33 * notice and this permission notice appear in all copies of the
34 * software, derivative works or modified versions, and any portions
35 * thereof, and that both notices appear in supporting documentation.
36 *
37 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
38 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
39 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
40 *
41 * Carnegie Mellon requests users of this software to return to
42 *
43 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
44 * School of Computer Science
45 * Carnegie Mellon University
46 * Pittsburgh PA 15213-3890
47 *
48 * any improvements or extensions that they make and grant Carnegie Mellon
49 * the rights to redistribute these changes.
50 */
51 /*
52 */
53 /*
54 * File: kern/zalloc.c
55 * Author: Avadis Tevanian, Jr.
56 *
57 * Zone-based memory allocator. A zone is a collection of fixed size
58 * data blocks for which quick allocation/deallocation is possible.
59 */
60 #include <zone_debug.h>
61 #include <norma_vm.h>
62 #include <mach_kdb.h>
63
64 #include <mach/mach_types.h>
65 #include <mach/vm_param.h>
66 #include <mach/kern_return.h>
67 #include <mach/mach_host_server.h>
68 #include <mach/machine/vm_types.h>
69 #include <mach_debug/zone_info.h>
70
71 #include <kern/kern_types.h>
72 #include <kern/assert.h>
73 #include <kern/host.h>
74 #include <kern/macro_help.h>
75 #include <kern/sched.h>
76 #include <kern/lock.h>
77 #include <kern/sched_prim.h>
78 #include <kern/misc_protos.h>
79 #include <kern/thread_call.h>
80 #include <kern/zalloc.h>
81 #include <kern/kalloc.h>
82
83 #include <vm/pmap.h>
84 #include <vm/vm_map.h>
85 #include <vm/vm_kern.h>
86 #include <vm/vm_page.h>
87
88 #include <machine/machparam.h>
89
90 #if defined(__ppc__)
91 /* for fake zone stat routines */
92 #include <ppc/savearea.h>
93 #include <ppc/mappings.h>
94 #endif
95
96 #if MACH_ASSERT
97 /* Detect use of zone elt after freeing it by two methods:
98 * (1) Range-check the free-list "next" ptr for sanity.
99 * (2) Store the ptr in two different words, and compare them against
100 * each other when re-using the zone elt, to detect modifications;
101 */
102
103 #if defined(__alpha)
104
105 #define is_kernel_data_addr(a) \
106 (!(a) || (IS_SYS_VA(a) && !((a) & (sizeof(long)-1))))
107
108 #else /* !defined(__alpha) */
109
110 #define is_kernel_data_addr(a) \
111 (!(a) || ((a) >= VM_MIN_KERNEL_ADDRESS && !((a) & 0x3)))
112
113 #endif /* defined(__alpha) */
114
115 /* Should we set all words of the zone element to an illegal address
116 * when it is freed, to help catch usage after freeing? The down-side
117 * is that this obscures the identity of the freed element.
118 */
119 boolean_t zfree_clear = FALSE;
120
121 #define ADD_TO_ZONE(zone, element) \
122 MACRO_BEGIN \
123 if (zfree_clear) \
124 { unsigned int i; \
125 for (i=1; \
126 i < zone->elem_size/sizeof(vm_offset_t) - 1; \
127 i++) \
128 ((vm_offset_t *)(element))[i] = 0xdeadbeef; \
129 } \
130 ((vm_offset_t *)(element))[0] = (zone)->free_elements; \
131 (zone)->free_elements = (vm_offset_t) (element); \
132 (zone)->count--; \
133 MACRO_END
134
135 #define REMOVE_FROM_ZONE(zone, ret, type) \
136 MACRO_BEGIN \
137 (ret) = (type) (zone)->free_elements; \
138 if ((ret) != (type) 0) { \
139 if (!is_kernel_data_addr(((vm_offset_t *)(ret))[0])) { \
140 panic("A freed zone element has been modified.\n"); \
141 } \
142 (zone)->count++; \
143 (zone)->free_elements = *((vm_offset_t *)(ret)); \
144 } \
145 MACRO_END
146 #else /* MACH_ASSERT */
147
148 #define ADD_TO_ZONE(zone, element) \
149 MACRO_BEGIN \
150 *((vm_offset_t *)(element)) = (zone)->free_elements; \
151 (zone)->free_elements = (vm_offset_t) (element); \
152 (zone)->count--; \
153 MACRO_END
154
155 #define REMOVE_FROM_ZONE(zone, ret, type) \
156 MACRO_BEGIN \
157 (ret) = (type) (zone)->free_elements; \
158 if ((ret) != (type) 0) { \
159 (zone)->count++; \
160 (zone)->free_elements = *((vm_offset_t *)(ret)); \
161 } \
162 MACRO_END
163
164 #endif /* MACH_ASSERT */
165
166 #if ZONE_DEBUG
167 #define zone_debug_enabled(z) z->active_zones.next
168 #define ROUNDUP(x,y) ((((x)+(y)-1)/(y))*(y))
169 #define ZONE_DEBUG_OFFSET ROUNDUP(sizeof(queue_chain_t),16)
170 #endif /* ZONE_DEBUG */
171
172 /*
173 * Support for garbage collection of unused zone pages:
174 */
175
176 struct zone_page_table_entry {
177 struct zone_page_table_entry *link;
178 short alloc_count;
179 short collect_count;
180 };
181
182 /* Forwards */
183 void zone_page_init(
184 vm_offset_t addr,
185 vm_size_t size,
186 int value);
187
188 void zone_page_alloc(
189 vm_offset_t addr,
190 vm_size_t size);
191
192 void zone_page_free_element(
193 struct zone_page_table_entry **free_pages,
194 vm_offset_t addr,
195 vm_size_t size);
196
197 void zone_page_collect(
198 vm_offset_t addr,
199 vm_size_t size);
200
201 boolean_t zone_page_collectable(
202 vm_offset_t addr,
203 vm_size_t size);
204
205 void zone_page_keep(
206 vm_offset_t addr,
207 vm_size_t size);
208
209 void zalloc_async(
210 thread_call_param_t p0,
211 thread_call_param_t p1);
212
213
214 #if ZONE_DEBUG && MACH_KDB
215 int zone_count(
216 zone_t z,
217 int tail);
218 #endif /* ZONE_DEBUG && MACH_KDB */
219
220 vm_map_t zone_map = VM_MAP_NULL;
221
222 zone_t zone_zone = ZONE_NULL; /* the zone containing other zones */
223
224 /*
225 * The VM system gives us an initial chunk of memory.
226 * It has to be big enough to allocate the zone_zone
227 */
228
229 vm_offset_t zdata;
230 vm_size_t zdata_size;
231
232 #define lock_zone(zone) \
233 MACRO_BEGIN \
234 mutex_lock(&(zone)->lock); \
235 MACRO_END
236
237 #define unlock_zone(zone) \
238 MACRO_BEGIN \
239 mutex_unlock(&(zone)->lock); \
240 MACRO_END
241
242 #define zone_wakeup(zone) thread_wakeup((event_t)(zone))
243 #define zone_sleep(zone) \
244 thread_sleep_mutex((event_t)(zone), \
245 &(zone)->lock, \
246 THREAD_UNINT)
247
248 #define lock_zone_init(zone) \
249 MACRO_BEGIN \
250 mutex_init(&zone->lock, 0); \
251 MACRO_END
252
253 #define lock_try_zone(zone) mutex_try(&zone->lock)
254
255 kern_return_t zget_space(
256 vm_offset_t size,
257 vm_offset_t *result);
258
259 decl_simple_lock_data(,zget_space_lock)
260 vm_offset_t zalloc_next_space;
261 vm_offset_t zalloc_end_of_space;
262 vm_size_t zalloc_wasted_space;
263
264 /*
265 * Garbage collection map information
266 */
267 struct zone_page_table_entry * zone_page_table;
268 vm_offset_t zone_map_min_address;
269 vm_offset_t zone_map_max_address;
270 unsigned int zone_pages;
271
272 /*
273 * Exclude more than one concurrent garbage collection
274 */
275 decl_mutex_data(, zone_gc_lock)
276
277 #define from_zone_map(addr, size) \
278 ((vm_offset_t)(addr) >= zone_map_min_address && \
279 ((vm_offset_t)(addr) + size -1) < zone_map_max_address)
280
281 #define ZONE_PAGE_USED 0
282 #define ZONE_PAGE_UNUSED -1
283
284
285 /*
286 * Protects first_zone, last_zone, num_zones,
287 * and the next_zone field of zones.
288 */
289 decl_simple_lock_data(, all_zones_lock)
290 zone_t first_zone;
291 zone_t *last_zone;
292 unsigned int num_zones;
293
294 boolean_t zone_gc_allowed = TRUE;
295 boolean_t zone_gc_forced = FALSE;
296 unsigned zone_gc_last_tick = 0;
297 unsigned zone_gc_max_rate = 0; /* in ticks */
298
299
300 /*
301 * zinit initializes a new zone. The zone data structures themselves
302 * are stored in a zone, which is initially a static structure that
303 * is initialized by zone_init.
304 */
305 zone_t
306 zinit(
307 vm_size_t size, /* the size of an element */
308 vm_size_t max, /* maximum memory to use */
309 vm_size_t alloc, /* allocation size */
310 const char *name) /* a name for the zone */
311 {
312 zone_t z;
313
314 if (zone_zone == ZONE_NULL) {
315 if (zget_space(sizeof(struct zone), (vm_offset_t *)&z)
316 != KERN_SUCCESS)
317 return(ZONE_NULL);
318 } else
319 z = (zone_t) zalloc(zone_zone);
320 if (z == ZONE_NULL)
321 return(ZONE_NULL);
322
323 /*
324 * Round off all the parameters appropriately.
325 */
326 if (size < sizeof(z->free_elements))
327 size = sizeof(z->free_elements);
328 size = ((size-1) + sizeof(z->free_elements)) -
329 ((size-1) % sizeof(z->free_elements));
330 if (alloc == 0)
331 alloc = PAGE_SIZE;
332 alloc = round_page(alloc);
333 max = round_page(max);
334 /*
335 * we look for an allocation size with less than 1% waste
336 * up to 5 pages in size...
337 * otherwise, we look for an allocation size with least fragmentation
338 * in the range of 1 - 5 pages
339 * This size will be used unless
340 * the user suggestion is larger AND has less fragmentation
341 */
342 { vm_size_t best, waste; unsigned int i;
343 best = PAGE_SIZE;
344 waste = best % size;
345
346 for (i = 1; i <= 5; i++) {
347 vm_size_t tsize, twaste;
348
349 tsize = i * PAGE_SIZE;
350
351 if ((tsize % size) < (tsize / 100)) {
352 alloc = tsize;
353 goto use_this_allocation;
354 }
355 twaste = tsize % size;
356 if (twaste < waste)
357 best = tsize, waste = twaste;
358 }
359 if (alloc <= best || (alloc % size >= waste))
360 alloc = best;
361 }
362 use_this_allocation:
363 if (max && (max < alloc))
364 max = alloc;
365
366 z->free_elements = 0;
367 z->cur_size = 0;
368 z->max_size = max;
369 z->elem_size = size;
370 z->alloc_size = alloc;
371 z->zone_name = name;
372 z->count = 0;
373 z->doing_alloc = FALSE;
374 z->doing_gc = FALSE;
375 z->exhaustible = FALSE;
376 z->collectable = TRUE;
377 z->allows_foreign = FALSE;
378 z->expandable = TRUE;
379 z->waiting = FALSE;
380 z->async_pending = FALSE;
381
382 #if ZONE_DEBUG
383 z->active_zones.next = z->active_zones.prev = 0;
384 zone_debug_enable(z);
385 #endif /* ZONE_DEBUG */
386 lock_zone_init(z);
387
388 /*
389 * Add the zone to the all-zones list.
390 */
391
392 z->next_zone = ZONE_NULL;
393 thread_call_setup(&z->call_async_alloc, zalloc_async, z);
394 simple_lock(&all_zones_lock);
395 *last_zone = z;
396 last_zone = &z->next_zone;
397 num_zones++;
398 simple_unlock(&all_zones_lock);
399
400 return(z);
401 }
402
403 /*
404 * Cram the given memory into the specified zone.
405 */
406 void
407 zcram(
408 register zone_t zone,
409 void *newaddr,
410 vm_size_t size)
411 {
412 register vm_size_t elem_size;
413 vm_offset_t newmem = (vm_offset_t) newaddr;
414
415 /* Basic sanity checks */
416 assert(zone != ZONE_NULL && newmem != (vm_offset_t)0);
417 assert(!zone->collectable || zone->allows_foreign
418 || (from_zone_map(newmem, size)));
419
420 elem_size = zone->elem_size;
421
422 lock_zone(zone);
423 while (size >= elem_size) {
424 ADD_TO_ZONE(zone, newmem);
425 if (from_zone_map(newmem, elem_size))
426 zone_page_alloc(newmem, elem_size);
427 zone->count++; /* compensate for ADD_TO_ZONE */
428 size -= elem_size;
429 newmem += elem_size;
430 zone->cur_size += elem_size;
431 }
432 unlock_zone(zone);
433 }
434
435 /*
436 * Contiguous space allocator for non-paged zones. Allocates "size" amount
437 * of memory from zone_map.
438 */
439
440 kern_return_t
441 zget_space(
442 vm_offset_t size,
443 vm_offset_t *result)
444 {
445 vm_offset_t new_space = 0;
446 vm_size_t space_to_add = 0;
447
448 simple_lock(&zget_space_lock);
449 while ((zalloc_next_space + size) > zalloc_end_of_space) {
450 /*
451 * Add at least one page to allocation area.
452 */
453
454 space_to_add = round_page(size);
455
456 if (new_space == 0) {
457 kern_return_t retval;
458 /*
459 * Memory cannot be wired down while holding
460 * any locks that the pageout daemon might
461 * need to free up pages. [Making the zget_space
462 * lock a complex lock does not help in this
463 * regard.]
464 *
465 * Unlock and allocate memory. Because several
466 * threads might try to do this at once, don't
467 * use the memory before checking for available
468 * space again.
469 */
470
471 simple_unlock(&zget_space_lock);
472
473 retval = kernel_memory_allocate(zone_map, &new_space,
474 space_to_add, 0, KMA_KOBJECT|KMA_NOPAGEWAIT);
475 if (retval != KERN_SUCCESS)
476 return(retval);
477 zone_page_init(new_space, space_to_add,
478 ZONE_PAGE_USED);
479 simple_lock(&zget_space_lock);
480 continue;
481 }
482
483
484 /*
485 * Memory was allocated in a previous iteration.
486 *
487 * Check whether the new region is contiguous
488 * with the old one.
489 */
490
491 if (new_space != zalloc_end_of_space) {
492 /*
493 * Throw away the remainder of the
494 * old space, and start a new one.
495 */
496 zalloc_wasted_space +=
497 zalloc_end_of_space - zalloc_next_space;
498 zalloc_next_space = new_space;
499 }
500
501 zalloc_end_of_space = new_space + space_to_add;
502
503 new_space = 0;
504 }
505 *result = zalloc_next_space;
506 zalloc_next_space += size;
507 simple_unlock(&zget_space_lock);
508
509 if (new_space != 0)
510 kmem_free(zone_map, new_space, space_to_add);
511
512 return(KERN_SUCCESS);
513 }
514
515
516 /*
517 * Steal memory for the zone package. Called from
518 * vm_page_bootstrap().
519 */
520 void
521 zone_steal_memory(void)
522 {
523 zdata_size = round_page(128*sizeof(struct zone));
524 zdata = (vm_offset_t)((char *)pmap_steal_memory(zdata_size) - (char *)0);
525 }
526
527
528 /*
529 * Fill a zone with enough memory to contain at least nelem elements.
530 * Memory is obtained with kmem_alloc_wired from the kernel_map.
531 * Return the number of elements actually put into the zone, which may
532 * be more than the caller asked for since the memory allocation is
533 * rounded up to a full page.
534 */
535 int
536 zfill(
537 zone_t zone,
538 int nelem)
539 {
540 kern_return_t kr;
541 vm_size_t size;
542 vm_offset_t memory;
543 int nalloc;
544
545 assert(nelem > 0);
546 if (nelem <= 0)
547 return 0;
548 size = nelem * zone->elem_size;
549 size = round_page(size);
550 kr = kmem_alloc_wired(kernel_map, &memory, size);
551 if (kr != KERN_SUCCESS)
552 return 0;
553
554 zone_change(zone, Z_FOREIGN, TRUE);
555 zcram(zone, (void *)memory, size);
556 nalloc = size / zone->elem_size;
557 assert(nalloc >= nelem);
558
559 return nalloc;
560 }
561
562 /*
563 * Initialize the "zone of zones" which uses fixed memory allocated
564 * earlier in memory initialization. zone_bootstrap is called
565 * before zone_init.
566 */
567 void
568 zone_bootstrap(void)
569 {
570 vm_size_t zone_zone_size;
571 vm_offset_t zone_zone_space;
572
573 simple_lock_init(&all_zones_lock, 0);
574
575 first_zone = ZONE_NULL;
576 last_zone = &first_zone;
577 num_zones = 0;
578
579 simple_lock_init(&zget_space_lock, 0);
580 zalloc_next_space = zdata;
581 zalloc_end_of_space = zdata + zdata_size;
582 zalloc_wasted_space = 0;
583
584 /* assertion: nobody else called zinit before us */
585 assert(zone_zone == ZONE_NULL);
586 zone_zone = zinit(sizeof(struct zone), 128 * sizeof(struct zone),
587 sizeof(struct zone), "zones");
588 zone_change(zone_zone, Z_COLLECT, FALSE);
589 zone_zone_size = zalloc_end_of_space - zalloc_next_space;
590 zget_space(zone_zone_size, &zone_zone_space);
591 zcram(zone_zone, (void *)zone_zone_space, zone_zone_size);
592 }
593
594 void
595 zone_init(
596 vm_size_t max_zonemap_size)
597 {
598 kern_return_t retval;
599 vm_offset_t zone_min;
600 vm_offset_t zone_max;
601 vm_size_t zone_table_size;
602
603 retval = kmem_suballoc(kernel_map, &zone_min, max_zonemap_size,
604 FALSE, VM_FLAGS_ANYWHERE, &zone_map);
605
606 if (retval != KERN_SUCCESS)
607 panic("zone_init: kmem_suballoc failed");
608 zone_max = zone_min + round_page(max_zonemap_size);
609 /*
610 * Setup garbage collection information:
611 */
612 zone_table_size = atop_32(zone_max - zone_min) *
613 sizeof(struct zone_page_table_entry);
614 if (kmem_alloc_wired(zone_map, (vm_offset_t *) &zone_page_table,
615 zone_table_size) != KERN_SUCCESS)
616 panic("zone_init");
617 zone_min = (vm_offset_t)zone_page_table + round_page(zone_table_size);
618 zone_pages = atop_32(zone_max - zone_min);
619 zone_map_min_address = zone_min;
620 zone_map_max_address = zone_max;
621 mutex_init(&zone_gc_lock, 0);
622 zone_page_init(zone_min, zone_max - zone_min, ZONE_PAGE_UNUSED);
623 }
624
625
626 /*
627 * zalloc returns an element from the specified zone.
628 */
629 void *
630 zalloc_canblock(
631 register zone_t zone,
632 boolean_t canblock)
633 {
634 vm_offset_t addr;
635 kern_return_t retval;
636
637 assert(zone != ZONE_NULL);
638
639 lock_zone(zone);
640
641 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
642
643 while ((addr == 0) && canblock && (zone->doing_gc)) {
644 zone->waiting = TRUE;
645 zone_sleep(zone);
646 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
647 }
648
649 while ((addr == 0) && canblock) {
650 /*
651 * If nothing was there, try to get more
652 */
653 if (zone->doing_alloc) {
654 /*
655 * Someone is allocating memory for this zone.
656 * Wait for it to show up, then try again.
657 */
658 zone->waiting = TRUE;
659 zone_sleep(zone);
660 }
661 else {
662 if ((zone->cur_size + zone->elem_size) >
663 zone->max_size) {
664 if (zone->exhaustible)
665 break;
666 if (zone->expandable) {
667 /*
668 * We're willing to overflow certain
669 * zones, but not without complaining.
670 *
671 * This is best used in conjunction
672 * with the collectable flag. What we
673 * want is an assurance we can get the
674 * memory back, assuming there's no
675 * leak.
676 */
677 zone->max_size += (zone->max_size >> 1);
678 } else {
679 unlock_zone(zone);
680
681 panic("zalloc: zone \"%s\" empty.", zone->zone_name);
682 }
683 }
684 zone->doing_alloc = TRUE;
685 unlock_zone(zone);
686
687 if (zone->collectable) {
688 vm_offset_t space;
689 vm_size_t alloc_size;
690 boolean_t retry = FALSE;
691
692 for (;;) {
693
694 if (vm_pool_low() || retry == TRUE)
695 alloc_size =
696 round_page(zone->elem_size);
697 else
698 alloc_size = zone->alloc_size;
699
700 retval = kernel_memory_allocate(zone_map,
701 &space, alloc_size, 0,
702 KMA_KOBJECT|KMA_NOPAGEWAIT);
703 if (retval == KERN_SUCCESS) {
704 zone_page_init(space, alloc_size,
705 ZONE_PAGE_USED);
706 zcram(zone, (void *)space, alloc_size);
707
708 break;
709 } else if (retval != KERN_RESOURCE_SHORTAGE) {
710 /* would like to cause a zone_gc() */
711 if (retry == TRUE)
712 panic("zalloc: \"%s\" (%d elements) retry fail %d", zone->zone_name, zone->count, retval);
713 retry = TRUE;
714 } else {
715 break;
716 }
717 }
718 lock_zone(zone);
719 zone->doing_alloc = FALSE;
720 if (zone->waiting) {
721 zone->waiting = FALSE;
722 zone_wakeup(zone);
723 }
724 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
725 if (addr == 0 &&
726 retval == KERN_RESOURCE_SHORTAGE) {
727 unlock_zone(zone);
728
729 VM_PAGE_WAIT();
730 lock_zone(zone);
731 }
732 } else {
733 vm_offset_t space;
734 retval = zget_space(zone->elem_size, &space);
735
736 lock_zone(zone);
737 zone->doing_alloc = FALSE;
738 if (zone->waiting) {
739 zone->waiting = FALSE;
740 thread_wakeup((event_t)zone);
741 }
742 if (retval == KERN_SUCCESS) {
743 zone->count++;
744 zone->cur_size += zone->elem_size;
745 #if ZONE_DEBUG
746 if (zone_debug_enabled(zone)) {
747 enqueue_tail(&zone->active_zones, (queue_entry_t)space);
748 }
749 #endif
750 unlock_zone(zone);
751 zone_page_alloc(space, zone->elem_size);
752 #if ZONE_DEBUG
753 if (zone_debug_enabled(zone))
754 space += ZONE_DEBUG_OFFSET;
755 #endif
756 return((void *)space);
757 }
758 if (retval == KERN_RESOURCE_SHORTAGE) {
759 unlock_zone(zone);
760
761 VM_PAGE_WAIT();
762 lock_zone(zone);
763 } else {
764 panic("zalloc: \"%s\" (%d elements) zget_space returned %d", zone->zone_name, zone->count, retval);
765 }
766 }
767 }
768 if (addr == 0)
769 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
770 }
771
772 if ((addr == 0) && !canblock && (zone->async_pending == FALSE) && (!vm_pool_low())) {
773 zone->async_pending = TRUE;
774 unlock_zone(zone);
775 thread_call_enter(&zone->call_async_alloc);
776 lock_zone(zone);
777 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
778 }
779
780 #if ZONE_DEBUG
781 if (addr && zone_debug_enabled(zone)) {
782 enqueue_tail(&zone->active_zones, (queue_entry_t)addr);
783 addr += ZONE_DEBUG_OFFSET;
784 }
785 #endif
786
787 unlock_zone(zone);
788
789 return((void *)addr);
790 }
791
792
793 void *
794 zalloc(
795 register zone_t zone)
796 {
797 return( zalloc_canblock(zone, TRUE) );
798 }
799
800 void *
801 zalloc_noblock(
802 register zone_t zone)
803 {
804 return( zalloc_canblock(zone, FALSE) );
805 }
806
807 void
808 zalloc_async(
809 thread_call_param_t p0,
810 __unused thread_call_param_t p1)
811 {
812 void *elt;
813
814 elt = zalloc_canblock((zone_t)p0, TRUE);
815 zfree((zone_t)p0, elt);
816 lock_zone(((zone_t)p0));
817 ((zone_t)p0)->async_pending = FALSE;
818 unlock_zone(((zone_t)p0));
819 }
820
821
822 /*
823 * zget returns an element from the specified zone
824 * and immediately returns nothing if there is nothing there.
825 *
826 * This form should be used when you can not block (like when
827 * processing an interrupt).
828 */
829 void *
830 zget(
831 register zone_t zone)
832 {
833 register vm_offset_t addr;
834
835 assert( zone != ZONE_NULL );
836
837 if (!lock_try_zone(zone))
838 return NULL;
839
840 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
841 #if ZONE_DEBUG
842 if (addr && zone_debug_enabled(zone)) {
843 enqueue_tail(&zone->active_zones, (queue_entry_t)addr);
844 addr += ZONE_DEBUG_OFFSET;
845 }
846 #endif /* ZONE_DEBUG */
847 unlock_zone(zone);
848
849 return((void *) addr);
850 }
851
852 /* Keep this FALSE by default. Large memory machine run orders of magnitude
853 slower in debug mode when true. Use debugger to enable if needed */
854 /* static */ boolean_t zone_check = FALSE;
855
856 static zone_t zone_last_bogus_zone = ZONE_NULL;
857 static vm_offset_t zone_last_bogus_elem = 0;
858
859 void
860 zfree(
861 register zone_t zone,
862 void *addr)
863 {
864 vm_offset_t elem = (vm_offset_t) addr;
865
866 #if MACH_ASSERT
867 /* Basic sanity checks */
868 if (zone == ZONE_NULL || elem == (vm_offset_t)0)
869 panic("zfree: NULL");
870 /* zone_gc assumes zones are never freed */
871 if (zone == zone_zone)
872 panic("zfree: freeing to zone_zone breaks zone_gc!");
873 #endif
874
875 if (zone->collectable && !zone->allows_foreign &&
876 !from_zone_map(elem, zone->elem_size)) {
877 #if MACH_ASSERT
878 panic("zfree: non-allocated memory in collectable zone!");
879 #endif
880 zone_last_bogus_zone = zone;
881 zone_last_bogus_elem = elem;
882 return;
883 }
884
885 lock_zone(zone);
886 #if ZONE_DEBUG
887 if (zone_debug_enabled(zone)) {
888 queue_t tmp_elem;
889
890 elem -= ZONE_DEBUG_OFFSET;
891 if (zone_check) {
892 /* check the zone's consistency */
893
894 for (tmp_elem = queue_first(&zone->active_zones);
895 !queue_end(tmp_elem, &zone->active_zones);
896 tmp_elem = queue_next(tmp_elem))
897 if (elem == (vm_offset_t)tmp_elem)
898 break;
899 if (elem != (vm_offset_t)tmp_elem)
900 panic("zfree()ing element from wrong zone");
901 }
902 remqueue(&zone->active_zones, (queue_t) elem);
903 }
904 #endif /* ZONE_DEBUG */
905 if (zone_check) {
906 vm_offset_t this;
907
908 /* check the zone's consistency */
909
910 for (this = zone->free_elements;
911 this != 0;
912 this = * (vm_offset_t *) this)
913 if (!pmap_kernel_va(this) || this == elem)
914 panic("zfree");
915 }
916 ADD_TO_ZONE(zone, elem);
917
918 /*
919 * If elements have one or more pages, and memory is low,
920 * request to run the garbage collection in the zone the next
921 * time the pageout thread runs.
922 */
923 if (zone->elem_size >= PAGE_SIZE &&
924 vm_pool_low()){
925 zone_gc_forced = TRUE;
926 }
927 unlock_zone(zone);
928 }
929
930
931 /* Change a zone's flags.
932 * This routine must be called immediately after zinit.
933 */
934 void
935 zone_change(
936 zone_t zone,
937 unsigned int item,
938 boolean_t value)
939 {
940 assert( zone != ZONE_NULL );
941 assert( value == TRUE || value == FALSE );
942
943 switch(item){
944 case Z_EXHAUST:
945 zone->exhaustible = value;
946 break;
947 case Z_COLLECT:
948 zone->collectable = value;
949 break;
950 case Z_EXPAND:
951 zone->expandable = value;
952 break;
953 case Z_FOREIGN:
954 zone->allows_foreign = value;
955 break;
956 #if MACH_ASSERT
957 default:
958 panic("Zone_change: Wrong Item Type!");
959 /* break; */
960 #endif
961 }
962 lock_zone_init(zone);
963 }
964
965 /*
966 * Return the expected number of free elements in the zone.
967 * This calculation will be incorrect if items are zfree'd that
968 * were never zalloc'd/zget'd. The correct way to stuff memory
969 * into a zone is by zcram.
970 */
971
972 integer_t
973 zone_free_count(zone_t zone)
974 {
975 integer_t free_count;
976
977 lock_zone(zone);
978 free_count = zone->cur_size/zone->elem_size - zone->count;
979 unlock_zone(zone);
980
981 assert(free_count >= 0);
982
983 return(free_count);
984 }
985
986 /*
987 * zprealloc preallocates wired memory, exanding the specified
988 * zone to the specified size
989 */
990 void
991 zprealloc(
992 zone_t zone,
993 vm_size_t size)
994 {
995 vm_offset_t addr;
996
997 if (size != 0) {
998 if (kmem_alloc_wired(zone_map, &addr, size) != KERN_SUCCESS)
999 panic("zprealloc");
1000 zone_page_init(addr, size, ZONE_PAGE_USED);
1001 zcram(zone, (void *)addr, size);
1002 }
1003 }
1004
1005 /*
1006 * Zone garbage collection subroutines
1007 */
1008
1009 boolean_t
1010 zone_page_collectable(
1011 vm_offset_t addr,
1012 vm_size_t size)
1013 {
1014 struct zone_page_table_entry *zp;
1015 natural_t i, j;
1016
1017 #if MACH_ASSERT
1018 if (!from_zone_map(addr, size))
1019 panic("zone_page_collectable");
1020 #endif
1021
1022 i = atop_32(addr-zone_map_min_address);
1023 j = atop_32((addr+size-1) - zone_map_min_address);
1024
1025 for (zp = zone_page_table + i; i <= j; zp++, i++)
1026 if (zp->collect_count == zp->alloc_count)
1027 return (TRUE);
1028
1029 return (FALSE);
1030 }
1031
1032 void
1033 zone_page_keep(
1034 vm_offset_t addr,
1035 vm_size_t size)
1036 {
1037 struct zone_page_table_entry *zp;
1038 natural_t i, j;
1039
1040 #if MACH_ASSERT
1041 if (!from_zone_map(addr, size))
1042 panic("zone_page_keep");
1043 #endif
1044
1045 i = atop_32(addr-zone_map_min_address);
1046 j = atop_32((addr+size-1) - zone_map_min_address);
1047
1048 for (zp = zone_page_table + i; i <= j; zp++, i++)
1049 zp->collect_count = 0;
1050 }
1051
1052 void
1053 zone_page_collect(
1054 vm_offset_t addr,
1055 vm_size_t size)
1056 {
1057 struct zone_page_table_entry *zp;
1058 natural_t i, j;
1059
1060 #if MACH_ASSERT
1061 if (!from_zone_map(addr, size))
1062 panic("zone_page_collect");
1063 #endif
1064
1065 i = atop_32(addr-zone_map_min_address);
1066 j = atop_32((addr+size-1) - zone_map_min_address);
1067
1068 for (zp = zone_page_table + i; i <= j; zp++, i++)
1069 ++zp->collect_count;
1070 }
1071
1072 void
1073 zone_page_init(
1074 vm_offset_t addr,
1075 vm_size_t size,
1076 int value)
1077 {
1078 struct zone_page_table_entry *zp;
1079 natural_t i, j;
1080
1081 #if MACH_ASSERT
1082 if (!from_zone_map(addr, size))
1083 panic("zone_page_init");
1084 #endif
1085
1086 i = atop_32(addr-zone_map_min_address);
1087 j = atop_32((addr+size-1) - zone_map_min_address);
1088
1089 for (zp = zone_page_table + i; i <= j; zp++, i++) {
1090 zp->alloc_count = value;
1091 zp->collect_count = 0;
1092 }
1093 }
1094
1095 void
1096 zone_page_alloc(
1097 vm_offset_t addr,
1098 vm_size_t size)
1099 {
1100 struct zone_page_table_entry *zp;
1101 natural_t i, j;
1102
1103 #if MACH_ASSERT
1104 if (!from_zone_map(addr, size))
1105 panic("zone_page_alloc");
1106 #endif
1107
1108 i = atop_32(addr-zone_map_min_address);
1109 j = atop_32((addr+size-1) - zone_map_min_address);
1110
1111 for (zp = zone_page_table + i; i <= j; zp++, i++) {
1112 /*
1113 * Set alloc_count to (ZONE_PAGE_USED + 1) if
1114 * it was previously set to ZONE_PAGE_UNUSED.
1115 */
1116 if (zp->alloc_count == ZONE_PAGE_UNUSED)
1117 zp->alloc_count = 1;
1118 else
1119 ++zp->alloc_count;
1120 }
1121 }
1122
1123 void
1124 zone_page_free_element(
1125 struct zone_page_table_entry **free_pages,
1126 vm_offset_t addr,
1127 vm_size_t size)
1128 {
1129 struct zone_page_table_entry *zp;
1130 natural_t i, j;
1131
1132 #if MACH_ASSERT
1133 if (!from_zone_map(addr, size))
1134 panic("zone_page_free_element");
1135 #endif
1136
1137 i = atop_32(addr-zone_map_min_address);
1138 j = atop_32((addr+size-1) - zone_map_min_address);
1139
1140 for (zp = zone_page_table + i; i <= j; zp++, i++) {
1141 if (zp->collect_count > 0)
1142 --zp->collect_count;
1143 if (--zp->alloc_count == 0) {
1144 zp->alloc_count = ZONE_PAGE_UNUSED;
1145 zp->collect_count = 0;
1146
1147 zp->link = *free_pages;
1148 *free_pages = zp;
1149 }
1150 }
1151 }
1152
1153
1154 /* This is used for walking through a zone's free element list.
1155 */
1156 struct zone_free_element {
1157 struct zone_free_element * next;
1158 };
1159
1160 struct {
1161 uint32_t pgs_freed;
1162
1163 uint32_t elems_collected,
1164 elems_freed,
1165 elems_kept;
1166 } zgc_stats;
1167
1168 /* Zone garbage collection
1169 *
1170 * zone_gc will walk through all the free elements in all the
1171 * zones that are marked collectable looking for reclaimable
1172 * pages. zone_gc is called by consider_zone_gc when the system
1173 * begins to run out of memory.
1174 */
1175 void
1176 zone_gc(void)
1177 {
1178 unsigned int max_zones;
1179 zone_t z;
1180 unsigned int i;
1181 struct zone_page_table_entry *zp, *zone_free_pages;
1182
1183 mutex_lock(&zone_gc_lock);
1184
1185 simple_lock(&all_zones_lock);
1186 max_zones = num_zones;
1187 z = first_zone;
1188 simple_unlock(&all_zones_lock);
1189
1190 #if MACH_ASSERT
1191 for (i = 0; i < zone_pages; i++)
1192 assert(zone_page_table[i].collect_count == 0);
1193 #endif /* MACH_ASSERT */
1194
1195 zone_free_pages = NULL;
1196
1197 for (i = 0; i < max_zones; i++, z = z->next_zone) {
1198 unsigned int n, m;
1199 vm_size_t elt_size, size_freed;
1200 struct zone_free_element *elt, *base_elt, *base_prev, *prev, *scan, *keep, *tail;
1201
1202 assert(z != ZONE_NULL);
1203
1204 if (!z->collectable)
1205 continue;
1206
1207 lock_zone(z);
1208
1209 elt_size = z->elem_size;
1210
1211 /*
1212 * Do a quick feasability check before we scan the zone:
1213 * skip unless there is likelihood of getting pages back
1214 * (i.e we need a whole allocation block's worth of free
1215 * elements before we can garbage collect) and
1216 * the zone has more than 10 percent of it's elements free
1217 */
1218 if (((z->cur_size - z->count * elt_size) <= (2 * z->alloc_size)) ||
1219 ((z->cur_size - z->count * elt_size) <= (z->cur_size / 10))) {
1220 unlock_zone(z);
1221 continue;
1222 }
1223
1224 z->doing_gc = TRUE;
1225
1226 /*
1227 * Snatch all of the free elements away from the zone.
1228 */
1229
1230 scan = (void *)z->free_elements;
1231 (void *)z->free_elements = NULL;
1232
1233 unlock_zone(z);
1234
1235 /*
1236 * Pass 1:
1237 *
1238 * Determine which elements we can attempt to collect
1239 * and count them up in the page table. Foreign elements
1240 * are returned to the zone.
1241 */
1242
1243 prev = (void *)&scan;
1244 elt = scan;
1245 n = 0; tail = keep = NULL;
1246 while (elt != NULL) {
1247 if (from_zone_map(elt, elt_size)) {
1248 zone_page_collect((vm_offset_t)elt, elt_size);
1249
1250 prev = elt;
1251 elt = elt->next;
1252
1253 ++zgc_stats.elems_collected;
1254 }
1255 else {
1256 if (keep == NULL)
1257 keep = tail = elt;
1258 else
1259 tail = tail->next = elt;
1260
1261 elt = prev->next = elt->next;
1262 tail->next = NULL;
1263 }
1264
1265 /*
1266 * Dribble back the elements we are keeping.
1267 */
1268
1269 if (++n >= 50) {
1270 if (z->waiting == TRUE) {
1271 lock_zone(z);
1272
1273 if (keep != NULL) {
1274 tail->next = (void *)z->free_elements;
1275 (void *)z->free_elements = keep;
1276 tail = keep = NULL;
1277 } else {
1278 m =0;
1279 base_elt = elt;
1280 base_prev = prev;
1281 while ((elt != NULL) && (++m < 50)) {
1282 prev = elt;
1283 elt = elt->next;
1284 }
1285 if (m !=0 ) {
1286 prev->next = (void *)z->free_elements;
1287 (void *)z->free_elements = (void *)base_elt;
1288 base_prev->next = elt;
1289 prev = base_prev;
1290 }
1291 }
1292
1293 if (z->waiting) {
1294 z->waiting = FALSE;
1295 zone_wakeup(z);
1296 }
1297
1298 unlock_zone(z);
1299 }
1300 n =0;
1301 }
1302 }
1303
1304 /*
1305 * Return any remaining elements.
1306 */
1307
1308 if (keep != NULL) {
1309 lock_zone(z);
1310
1311 tail->next = (void *)z->free_elements;
1312 (void *)z->free_elements = keep;
1313
1314 unlock_zone(z);
1315 }
1316
1317 /*
1318 * Pass 2:
1319 *
1320 * Determine which pages we can reclaim and
1321 * free those elements.
1322 */
1323
1324 size_freed = 0;
1325 prev = (void *)&scan;
1326 elt = scan;
1327 n = 0; tail = keep = NULL;
1328 while (elt != NULL) {
1329 if (zone_page_collectable((vm_offset_t)elt, elt_size)) {
1330 size_freed += elt_size;
1331 zone_page_free_element(&zone_free_pages,
1332 (vm_offset_t)elt, elt_size);
1333
1334 elt = prev->next = elt->next;
1335
1336 ++zgc_stats.elems_freed;
1337 }
1338 else {
1339 zone_page_keep((vm_offset_t)elt, elt_size);
1340
1341 if (keep == NULL)
1342 keep = tail = elt;
1343 else
1344 tail = tail->next = elt;
1345
1346 elt = prev->next = elt->next;
1347 tail->next = NULL;
1348
1349 ++zgc_stats.elems_kept;
1350 }
1351
1352 /*
1353 * Dribble back the elements we are keeping,
1354 * and update the zone size info.
1355 */
1356
1357 if (++n >= 50) {
1358 lock_zone(z);
1359
1360 z->cur_size -= size_freed;
1361 size_freed = 0;
1362
1363 if (keep != NULL) {
1364 tail->next = (void *)z->free_elements;
1365 (void *)z->free_elements = keep;
1366 }
1367
1368 if (z->waiting) {
1369 z->waiting = FALSE;
1370 zone_wakeup(z);
1371 }
1372
1373 unlock_zone(z);
1374
1375 n = 0; tail = keep = NULL;
1376 }
1377 }
1378
1379 /*
1380 * Return any remaining elements, and update
1381 * the zone size info.
1382 */
1383
1384 lock_zone(z);
1385
1386 if (size_freed > 0 || keep != NULL) {
1387
1388 z->cur_size -= size_freed;
1389
1390 if (keep != NULL) {
1391 tail->next = (void *)z->free_elements;
1392 (void *)z->free_elements = keep;
1393 }
1394
1395 }
1396
1397 z->doing_gc = FALSE;
1398 if (z->waiting) {
1399 z->waiting = FALSE;
1400 zone_wakeup(z);
1401 }
1402 unlock_zone(z);
1403 }
1404
1405 /*
1406 * Reclaim the pages we are freeing.
1407 */
1408
1409 while ((zp = zone_free_pages) != NULL) {
1410 zone_free_pages = zp->link;
1411 kmem_free(zone_map, zone_map_min_address + PAGE_SIZE *
1412 (zp - zone_page_table), PAGE_SIZE);
1413 ++zgc_stats.pgs_freed;
1414 }
1415
1416 mutex_unlock(&zone_gc_lock);
1417 }
1418
1419 /*
1420 * consider_zone_gc:
1421 *
1422 * Called by the pageout daemon when the system needs more free pages.
1423 */
1424
1425 void
1426 consider_zone_gc(void)
1427 {
1428 /*
1429 * By default, don't attempt zone GC more frequently
1430 * than once / 1 minutes.
1431 */
1432
1433 if (zone_gc_max_rate == 0)
1434 zone_gc_max_rate = (60 << SCHED_TICK_SHIFT) + 1;
1435
1436 if (zone_gc_allowed &&
1437 ((sched_tick > (zone_gc_last_tick + zone_gc_max_rate)) ||
1438 zone_gc_forced)) {
1439 zone_gc_forced = FALSE;
1440 zone_gc_last_tick = sched_tick;
1441 zone_gc();
1442 }
1443 }
1444
1445
1446 kern_return_t
1447 host_zone_info(
1448 host_t host,
1449 zone_name_array_t *namesp,
1450 mach_msg_type_number_t *namesCntp,
1451 zone_info_array_t *infop,
1452 mach_msg_type_number_t *infoCntp)
1453 {
1454 zone_name_t *names;
1455 vm_offset_t names_addr;
1456 vm_size_t names_size;
1457 zone_info_t *info;
1458 vm_offset_t info_addr;
1459 vm_size_t info_size;
1460 unsigned int max_zones, i;
1461 zone_t z;
1462 zone_name_t *zn;
1463 zone_info_t *zi;
1464 kern_return_t kr;
1465
1466 if (host == HOST_NULL)
1467 return KERN_INVALID_HOST;
1468
1469 /*
1470 * We assume that zones aren't freed once allocated.
1471 * We won't pick up any zones that are allocated later.
1472 */
1473
1474 simple_lock(&all_zones_lock);
1475 #ifdef ppc
1476 max_zones = num_zones + 4;
1477 #else
1478 max_zones = num_zones + 2;
1479 #endif
1480 z = first_zone;
1481 simple_unlock(&all_zones_lock);
1482
1483 if (max_zones <= *namesCntp) {
1484 /* use in-line memory */
1485 names_size = *namesCntp * sizeof *names;
1486 names = *namesp;
1487 } else {
1488 names_size = round_page(max_zones * sizeof *names);
1489 kr = kmem_alloc_pageable(ipc_kernel_map,
1490 &names_addr, names_size);
1491 if (kr != KERN_SUCCESS)
1492 return kr;
1493 names = (zone_name_t *) names_addr;
1494 }
1495
1496 if (max_zones <= *infoCntp) {
1497 /* use in-line memory */
1498 info_size = *infoCntp * sizeof *info;
1499 info = *infop;
1500 } else {
1501 info_size = round_page(max_zones * sizeof *info);
1502 kr = kmem_alloc_pageable(ipc_kernel_map,
1503 &info_addr, info_size);
1504 if (kr != KERN_SUCCESS) {
1505 if (names != *namesp)
1506 kmem_free(ipc_kernel_map,
1507 names_addr, names_size);
1508 return kr;
1509 }
1510
1511 info = (zone_info_t *) info_addr;
1512 }
1513 zn = &names[0];
1514 zi = &info[0];
1515
1516 for (i = 0; i < num_zones; i++) {
1517 struct zone zcopy;
1518
1519 assert(z != ZONE_NULL);
1520
1521 lock_zone(z);
1522 zcopy = *z;
1523 unlock_zone(z);
1524
1525 simple_lock(&all_zones_lock);
1526 z = z->next_zone;
1527 simple_unlock(&all_zones_lock);
1528
1529 /* assuming here the name data is static */
1530 (void) strncpy(zn->zn_name, zcopy.zone_name,
1531 sizeof zn->zn_name);
1532
1533 zi->zi_count = zcopy.count;
1534 zi->zi_cur_size = zcopy.cur_size;
1535 zi->zi_max_size = zcopy.max_size;
1536 zi->zi_elem_size = zcopy.elem_size;
1537 zi->zi_alloc_size = zcopy.alloc_size;
1538 zi->zi_exhaustible = zcopy.exhaustible;
1539 zi->zi_collectable = zcopy.collectable;
1540
1541 zn++;
1542 zi++;
1543 }
1544 strcpy(zn->zn_name, "kernel_stacks");
1545 stack_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
1546 &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
1547 zn++;
1548 zi++;
1549 #ifdef ppc
1550 strcpy(zn->zn_name, "save_areas");
1551 save_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
1552 &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
1553 zn++;
1554 zi++;
1555
1556 strcpy(zn->zn_name, "pmap_mappings");
1557 mapping_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
1558 &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
1559 zn++;
1560 zi++;
1561 #endif
1562 strcpy(zn->zn_name, "kalloc.large");
1563 kalloc_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
1564 &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
1565
1566 if (names != *namesp) {
1567 vm_size_t used;
1568 vm_map_copy_t copy;
1569
1570 used = max_zones * sizeof *names;
1571
1572 if (used != names_size)
1573 bzero((char *) (names_addr + used), names_size - used);
1574
1575 kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)names_addr,
1576 (vm_map_size_t)names_size, TRUE, &copy);
1577 assert(kr == KERN_SUCCESS);
1578
1579 *namesp = (zone_name_t *) copy;
1580 }
1581 *namesCntp = max_zones;
1582
1583 if (info != *infop) {
1584 vm_size_t used;
1585 vm_map_copy_t copy;
1586
1587 used = max_zones * sizeof *info;
1588
1589 if (used != info_size)
1590 bzero((char *) (info_addr + used), info_size - used);
1591
1592 kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)info_addr,
1593 (vm_map_size_t)info_size, TRUE, &copy);
1594 assert(kr == KERN_SUCCESS);
1595
1596 *infop = (zone_info_t *) copy;
1597 }
1598 *infoCntp = max_zones;
1599
1600 return KERN_SUCCESS;
1601 }
1602
1603 #if MACH_KDB
1604 #include <ddb/db_command.h>
1605 #include <ddb/db_output.h>
1606 #include <kern/kern_print.h>
1607
1608 const char *zone_labels =
1609 "ENTRY COUNT TOT_SZ MAX_SZ ELT_SZ ALLOC_SZ NAME";
1610
1611 /* Forwards */
1612 void db_print_zone(
1613 zone_t addr);
1614
1615 #if ZONE_DEBUG
1616 void db_zone_check_active(
1617 zone_t zone);
1618 void db_zone_print_active(
1619 zone_t zone);
1620 #endif /* ZONE_DEBUG */
1621 void db_zone_print_free(
1622 zone_t zone);
1623 void
1624 db_print_zone(
1625 zone_t addr)
1626 {
1627 struct zone zcopy;
1628
1629 zcopy = *addr;
1630
1631 db_printf("%8x %8x %8x %8x %6x %8x %s ",
1632 addr, zcopy.count, zcopy.cur_size,
1633 zcopy.max_size, zcopy.elem_size,
1634 zcopy.alloc_size, zcopy.zone_name);
1635 if (zcopy.exhaustible)
1636 db_printf("H");
1637 if (zcopy.collectable)
1638 db_printf("C");
1639 if (zcopy.expandable)
1640 db_printf("X");
1641 db_printf("\n");
1642 }
1643
1644 /*ARGSUSED*/
1645 void
1646 db_show_one_zone(
1647 db_expr_t addr,
1648 int have_addr,
1649 __unused db_expr_t count,
1650 __unused char * modif)
1651 {
1652 struct zone *z = (zone_t)((char *)0 + addr);
1653
1654 if (z == ZONE_NULL || !have_addr){
1655 db_error("No Zone\n");
1656 /*NOTREACHED*/
1657 }
1658
1659 db_printf("%s\n", zone_labels);
1660 db_print_zone(z);
1661 }
1662
1663 /*ARGSUSED*/
1664 void
1665 db_show_all_zones(
1666 __unused db_expr_t addr,
1667 int have_addr,
1668 db_expr_t count,
1669 __unused char * modif)
1670 {
1671 zone_t z;
1672 unsigned total = 0;
1673
1674 /*
1675 * Don't risk hanging by unconditionally locking,
1676 * risk of incoherent data is small (zones aren't freed).
1677 */
1678 have_addr = simple_lock_try(&all_zones_lock);
1679 count = num_zones;
1680 z = first_zone;
1681 if (have_addr) {
1682 simple_unlock(&all_zones_lock);
1683 }
1684
1685 db_printf("%s\n", zone_labels);
1686 for ( ; count > 0; count--) {
1687 if (!z) {
1688 db_error("Mangled Zone List\n");
1689 /*NOTREACHED*/
1690 }
1691 db_print_zone(z);
1692 total += z->cur_size,
1693
1694 have_addr = simple_lock_try(&all_zones_lock);
1695 z = z->next_zone;
1696 if (have_addr) {
1697 simple_unlock(&all_zones_lock);
1698 }
1699 }
1700 db_printf("\nTotal %8x", total);
1701 db_printf("\n\nzone_gc() has reclaimed %d pages\n", zgc_stats.pgs_freed);
1702 }
1703
1704 #if ZONE_DEBUG
1705 void
1706 db_zone_check_active(
1707 zone_t zone)
1708 {
1709 int count = 0;
1710 queue_t tmp_elem;
1711
1712 if (!zone_debug_enabled(zone) || !zone_check)
1713 return;
1714 tmp_elem = queue_first(&zone->active_zones);
1715 while (count < zone->count) {
1716 count++;
1717 if (tmp_elem == 0) {
1718 printf("unexpected zero element, zone=0x%x, count=%d\n",
1719 zone, count);
1720 assert(FALSE);
1721 break;
1722 }
1723 if (queue_end(tmp_elem, &zone->active_zones)) {
1724 printf("unexpected queue_end, zone=0x%x, count=%d\n",
1725 zone, count);
1726 assert(FALSE);
1727 break;
1728 }
1729 tmp_elem = queue_next(tmp_elem);
1730 }
1731 if (!queue_end(tmp_elem, &zone->active_zones)) {
1732 printf("not at queue_end, zone=0x%x, tmp_elem=0x%x\n",
1733 zone, tmp_elem);
1734 assert(FALSE);
1735 }
1736 }
1737
1738 void
1739 db_zone_print_active(
1740 zone_t zone)
1741 {
1742 int count = 0;
1743 queue_t tmp_elem;
1744
1745 if (!zone_debug_enabled(zone)) {
1746 printf("zone 0x%x debug not enabled\n", zone);
1747 return;
1748 }
1749 if (!zone_check) {
1750 printf("zone_check FALSE\n");
1751 return;
1752 }
1753
1754 printf("zone 0x%x, active elements %d\n", zone, zone->count);
1755 printf("active list:\n");
1756 tmp_elem = queue_first(&zone->active_zones);
1757 while (count < zone->count) {
1758 printf(" 0x%x", tmp_elem);
1759 count++;
1760 if ((count % 6) == 0)
1761 printf("\n");
1762 if (tmp_elem == 0) {
1763 printf("\nunexpected zero element, count=%d\n", count);
1764 break;
1765 }
1766 if (queue_end(tmp_elem, &zone->active_zones)) {
1767 printf("\nunexpected queue_end, count=%d\n", count);
1768 break;
1769 }
1770 tmp_elem = queue_next(tmp_elem);
1771 }
1772 if (!queue_end(tmp_elem, &zone->active_zones))
1773 printf("\nnot at queue_end, tmp_elem=0x%x\n", tmp_elem);
1774 else
1775 printf("\n");
1776 }
1777 #endif /* ZONE_DEBUG */
1778
1779 void
1780 db_zone_print_free(
1781 zone_t zone)
1782 {
1783 int count = 0;
1784 int freecount;
1785 vm_offset_t elem;
1786
1787 freecount = zone_free_count(zone);
1788 printf("zone 0x%x, free elements %d\n", zone, freecount);
1789 printf("free list:\n");
1790 elem = zone->free_elements;
1791 while (count < freecount) {
1792 printf(" 0x%x", elem);
1793 count++;
1794 if ((count % 6) == 0)
1795 printf("\n");
1796 if (elem == 0) {
1797 printf("\nunexpected zero element, count=%d\n", count);
1798 break;
1799 }
1800 elem = *((vm_offset_t *)elem);
1801 }
1802 if (elem != 0)
1803 printf("\nnot at end of free list, elem=0x%x\n", elem);
1804 else
1805 printf("\n");
1806 }
1807
1808 #endif /* MACH_KDB */
1809
1810
1811 #if ZONE_DEBUG
1812
1813 /* should we care about locks here ? */
1814
1815 #if MACH_KDB
1816 void *
1817 next_element(
1818 zone_t z,
1819 void *prev)
1820 {
1821 char *elt = (char *)prev;
1822
1823 if (!zone_debug_enabled(z))
1824 return(0);
1825 elt -= ZONE_DEBUG_OFFSET;
1826 elt = (char *) queue_next((queue_t) elt);
1827 if ((queue_t) elt == &z->active_zones)
1828 return(0);
1829 elt += ZONE_DEBUG_OFFSET;
1830 return(elt);
1831 }
1832
1833 void *
1834 first_element(
1835 zone_t z)
1836 {
1837 char *elt;
1838
1839 if (!zone_debug_enabled(z))
1840 return(0);
1841 if (queue_empty(&z->active_zones))
1842 return(0);
1843 elt = (char *)queue_first(&z->active_zones);
1844 elt += ZONE_DEBUG_OFFSET;
1845 return(elt);
1846 }
1847
1848 /*
1849 * Second arg controls how many zone elements are printed:
1850 * 0 => none
1851 * n, n < 0 => all
1852 * n, n > 0 => last n on active list
1853 */
1854 int
1855 zone_count(
1856 zone_t z,
1857 int tail)
1858 {
1859 void *elt;
1860 int count = 0;
1861 boolean_t print = (tail != 0);
1862
1863 if (tail < 0)
1864 tail = z->count;
1865 if (z->count < tail)
1866 tail = 0;
1867 tail = z->count - tail;
1868 for (elt = first_element(z); elt; elt = next_element(z, elt)) {
1869 if (print && tail <= count)
1870 db_printf("%8x\n", elt);
1871 count++;
1872 }
1873 assert(count == z->count);
1874 return(count);
1875 }
1876 #endif /* MACH_KDB */
1877
1878 #define zone_in_use(z) ( z->count || z->free_elements )
1879
1880 void
1881 zone_debug_enable(
1882 zone_t z)
1883 {
1884 if (zone_debug_enabled(z) || zone_in_use(z) ||
1885 z->alloc_size < (z->elem_size + ZONE_DEBUG_OFFSET))
1886 return;
1887 queue_init(&z->active_zones);
1888 z->elem_size += ZONE_DEBUG_OFFSET;
1889 }
1890
1891 void
1892 zone_debug_disable(
1893 zone_t z)
1894 {
1895 if (!zone_debug_enabled(z) || zone_in_use(z))
1896 return;
1897 z->elem_size -= ZONE_DEBUG_OFFSET;
1898 z->active_zones.next = z->active_zones.prev = 0;
1899 }
1900 #endif /* ZONE_DEBUG */
mirror of XNU