]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/zalloc.c
projects / apple / xnu.git / blob
? search:


45de75913111ecc24c5c755f54fa4aafab11d8bb
[apple/xnu.git] / osfmk / kern / zalloc.c
1 /*
2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: kern/zalloc.c
60 * Author: Avadis Tevanian, Jr.
61 *
62 * Zone-based memory allocator. A zone is a collection of fixed size
63 * data blocks for which quick allocation/deallocation is possible.
64 */
65 #include <zone_debug.h>
66 #include <norma_vm.h>
67 #include <mach_kdb.h>
68
69 #include <mach/mach_types.h>
70 #include <mach/vm_param.h>
71 #include <mach/kern_return.h>
72 #include <mach/mach_host_server.h>
73 #include <mach/machine/vm_types.h>
74 #include <mach_debug/zone_info.h>
75
76 #include <kern/kern_types.h>
77 #include <kern/assert.h>
78 #include <kern/host.h>
79 #include <kern/macro_help.h>
80 #include <kern/sched.h>
81 #include <kern/lock.h>
82 #include <kern/sched_prim.h>
83 #include <kern/misc_protos.h>
84 #include <kern/thread_call.h>
85 #include <kern/zalloc.h>
86 #include <kern/kalloc.h>
87
88 #include <vm/pmap.h>
89 #include <vm/vm_map.h>
90 #include <vm/vm_kern.h>
91 #include <vm/vm_page.h>
92
93 #include <machine/machparam.h>
94
95 #if defined(__ppc__)
96 /* for fake zone stat routines */
97 #include <ppc/savearea.h>
98 #include <ppc/mappings.h>
99 #endif
100
101 #if MACH_ASSERT
102 /* Detect use of zone elt after freeing it by two methods:
103 * (1) Range-check the free-list "next" ptr for sanity.
104 * (2) Store the ptr in two different words, and compare them against
105 * each other when re-using the zone elt, to detect modifications;
106 */
107
108 #if defined(__alpha)
109
110 #define is_kernel_data_addr(a) \
111 (!(a) || (IS_SYS_VA(a) && !((a) & (sizeof(long)-1))))
112
113 #else /* !defined(__alpha) */
114
115 #define is_kernel_data_addr(a) \
116 (!(a) || ((a) >= vm_min_kernel_address && !((a) & 0x3)))
117
118 #endif /* defined(__alpha) */
119
120 /* Should we set all words of the zone element to an illegal address
121 * when it is freed, to help catch usage after freeing? The down-side
122 * is that this obscures the identity of the freed element.
123 */
124 boolean_t zfree_clear = FALSE;
125
126 #define ADD_TO_ZONE(zone, element) \
127 MACRO_BEGIN \
128 if (zfree_clear) \
129 { unsigned int i; \
130 for (i=1; \
131 i < zone->elem_size/sizeof(vm_offset_t) - 1; \
132 i++) \
133 ((vm_offset_t *)(element))[i] = 0xdeadbeef; \
134 } \
135 ((vm_offset_t *)(element))[0] = (zone)->free_elements; \
136 (zone)->free_elements = (vm_offset_t) (element); \
137 (zone)->count--; \
138 MACRO_END
139
140 #define REMOVE_FROM_ZONE(zone, ret, type) \
141 MACRO_BEGIN \
142 (ret) = (type) (zone)->free_elements; \
143 if ((ret) != (type) 0) { \
144 if (!is_kernel_data_addr(((vm_offset_t *)(ret))[0])) { \
145 panic("A freed zone element has been modified.\n"); \
146 } \
147 (zone)->count++; \
148 (zone)->free_elements = *((vm_offset_t *)(ret)); \
149 } \
150 MACRO_END
151 #else /* MACH_ASSERT */
152
153 #define ADD_TO_ZONE(zone, element) \
154 MACRO_BEGIN \
155 *((vm_offset_t *)(element)) = (zone)->free_elements; \
156 (zone)->free_elements = (vm_offset_t) (element); \
157 (zone)->count--; \
158 MACRO_END
159
160 #define REMOVE_FROM_ZONE(zone, ret, type) \
161 MACRO_BEGIN \
162 (ret) = (type) (zone)->free_elements; \
163 if ((ret) != (type) 0) { \
164 (zone)->count++; \
165 (zone)->free_elements = *((vm_offset_t *)(ret)); \
166 } \
167 MACRO_END
168
169 #endif /* MACH_ASSERT */
170
171 #if ZONE_DEBUG
172 #define zone_debug_enabled(z) z->active_zones.next
173 #define ROUNDUP(x,y) ((((x)+(y)-1)/(y))*(y))
174 #define ZONE_DEBUG_OFFSET ROUNDUP(sizeof(queue_chain_t),16)
175 #endif /* ZONE_DEBUG */
176
177 /*
178 * Support for garbage collection of unused zone pages:
179 */
180
181 struct zone_page_table_entry {
182 struct zone_page_table_entry *link;
183 short alloc_count;
184 short collect_count;
185 };
186
187 /* Forwards */
188 void zone_page_init(
189 vm_offset_t addr,
190 vm_size_t size,
191 int value);
192
193 void zone_page_alloc(
194 vm_offset_t addr,
195 vm_size_t size);
196
197 void zone_page_free_element(
198 struct zone_page_table_entry **free_pages,
199 vm_offset_t addr,
200 vm_size_t size);
201
202 void zone_page_collect(
203 vm_offset_t addr,
204 vm_size_t size);
205
206 boolean_t zone_page_collectable(
207 vm_offset_t addr,
208 vm_size_t size);
209
210 void zone_page_keep(
211 vm_offset_t addr,
212 vm_size_t size);
213
214 void zalloc_async(
215 thread_call_param_t p0,
216 thread_call_param_t p1);
217
218
219 #if ZONE_DEBUG && MACH_KDB
220 int zone_count(
221 zone_t z,
222 int tail);
223 #endif /* ZONE_DEBUG && MACH_KDB */
224
225 vm_map_t zone_map = VM_MAP_NULL;
226
227 zone_t zone_zone = ZONE_NULL; /* the zone containing other zones */
228
229 /*
230 * The VM system gives us an initial chunk of memory.
231 * It has to be big enough to allocate the zone_zone
232 */
233
234 vm_offset_t zdata;
235 vm_size_t zdata_size;
236
237 #define lock_zone(zone) \
238 MACRO_BEGIN \
239 mutex_lock(&(zone)->lock); \
240 MACRO_END
241
242 #define unlock_zone(zone) \
243 MACRO_BEGIN \
244 mutex_unlock(&(zone)->lock); \
245 MACRO_END
246
247 #define zone_wakeup(zone) thread_wakeup((event_t)(zone))
248 #define zone_sleep(zone) \
249 thread_sleep_mutex((event_t)(zone), \
250 &(zone)->lock, \
251 THREAD_UNINT)
252
253 #define lock_zone_init(zone) \
254 MACRO_BEGIN \
255 mutex_init(&zone->lock, 0); \
256 MACRO_END
257
258 #define lock_try_zone(zone) mutex_try(&zone->lock)
259
260 kern_return_t zget_space(
261 vm_offset_t size,
262 vm_offset_t *result);
263
264 decl_simple_lock_data(,zget_space_lock)
265 vm_offset_t zalloc_next_space;
266 vm_offset_t zalloc_end_of_space;
267 vm_size_t zalloc_wasted_space;
268
269 /*
270 * Garbage collection map information
271 */
272 struct zone_page_table_entry * zone_page_table;
273 vm_offset_t zone_map_min_address;
274 vm_offset_t zone_map_max_address;
275 unsigned int zone_pages;
276
277 /*
278 * Exclude more than one concurrent garbage collection
279 */
280 decl_mutex_data(, zone_gc_lock)
281
282 #define from_zone_map(addr, size) \
283 ((vm_offset_t)(addr) >= zone_map_min_address && \
284 ((vm_offset_t)(addr) + size -1) < zone_map_max_address)
285
286 #define ZONE_PAGE_USED 0
287 #define ZONE_PAGE_UNUSED -1
288
289
290 /*
291 * Protects first_zone, last_zone, num_zones,
292 * and the next_zone field of zones.
293 */
294 decl_simple_lock_data(, all_zones_lock)
295 zone_t first_zone;
296 zone_t *last_zone;
297 unsigned int num_zones;
298
299 boolean_t zone_gc_allowed = TRUE;
300 boolean_t zone_gc_forced = FALSE;
301 unsigned zone_gc_last_tick = 0;
302 unsigned zone_gc_max_rate = 0; /* in ticks */
303
304
305 /*
306 * zinit initializes a new zone. The zone data structures themselves
307 * are stored in a zone, which is initially a static structure that
308 * is initialized by zone_init.
309 */
310 zone_t
311 zinit(
312 vm_size_t size, /* the size of an element */
313 vm_size_t max, /* maximum memory to use */
314 vm_size_t alloc, /* allocation size */
315 const char *name) /* a name for the zone */
316 {
317 zone_t z;
318
319 if (zone_zone == ZONE_NULL) {
320 if (zget_space(sizeof(struct zone), (vm_offset_t *)&z)
321 != KERN_SUCCESS)
322 return(ZONE_NULL);
323 } else
324 z = (zone_t) zalloc(zone_zone);
325 if (z == ZONE_NULL)
326 return(ZONE_NULL);
327
328 /*
329 * Round off all the parameters appropriately.
330 */
331 if (size < sizeof(z->free_elements))
332 size = sizeof(z->free_elements);
333 size = ((size-1) + sizeof(z->free_elements)) -
334 ((size-1) % sizeof(z->free_elements));
335 if (alloc == 0)
336 alloc = PAGE_SIZE;
337 alloc = round_page(alloc);
338 max = round_page(max);
339 /*
340 * we look for an allocation size with less than 1% waste
341 * up to 5 pages in size...
342 * otherwise, we look for an allocation size with least fragmentation
343 * in the range of 1 - 5 pages
344 * This size will be used unless
345 * the user suggestion is larger AND has less fragmentation
346 */
347 { vm_size_t best, waste; unsigned int i;
348 best = PAGE_SIZE;
349 waste = best % size;
350
351 for (i = 1; i <= 5; i++) {
352 vm_size_t tsize, twaste;
353
354 tsize = i * PAGE_SIZE;
355
356 if ((tsize % size) < (tsize / 100)) {
357 alloc = tsize;
358 goto use_this_allocation;
359 }
360 twaste = tsize % size;
361 if (twaste < waste)
362 best = tsize, waste = twaste;
363 }
364 if (alloc <= best || (alloc % size >= waste))
365 alloc = best;
366 }
367 use_this_allocation:
368 if (max && (max < alloc))
369 max = alloc;
370
371 z->free_elements = 0;
372 z->cur_size = 0;
373 z->max_size = max;
374 z->elem_size = size;
375 z->alloc_size = alloc;
376 z->zone_name = name;
377 z->count = 0;
378 z->doing_alloc = FALSE;
379 z->doing_gc = FALSE;
380 z->exhaustible = FALSE;
381 z->collectable = TRUE;
382 z->allows_foreign = FALSE;
383 z->expandable = TRUE;
384 z->waiting = FALSE;
385 z->async_pending = FALSE;
386
387 #if ZONE_DEBUG
388 z->active_zones.next = z->active_zones.prev = 0;
389 zone_debug_enable(z);
390 #endif /* ZONE_DEBUG */
391 lock_zone_init(z);
392
393 /*
394 * Add the zone to the all-zones list.
395 */
396
397 z->next_zone = ZONE_NULL;
398 thread_call_setup(&z->call_async_alloc, zalloc_async, z);
399 simple_lock(&all_zones_lock);
400 *last_zone = z;
401 last_zone = &z->next_zone;
402 num_zones++;
403 simple_unlock(&all_zones_lock);
404
405 return(z);
406 }
407
408 /*
409 * Cram the given memory into the specified zone.
410 */
411 void
412 zcram(
413 register zone_t zone,
414 void *newaddr,
415 vm_size_t size)
416 {
417 register vm_size_t elem_size;
418 vm_offset_t newmem = (vm_offset_t) newaddr;
419
420 /* Basic sanity checks */
421 assert(zone != ZONE_NULL && newmem != (vm_offset_t)0);
422 assert(!zone->collectable || zone->allows_foreign
423 || (from_zone_map(newmem, size)));
424
425 elem_size = zone->elem_size;
426
427 lock_zone(zone);
428 while (size >= elem_size) {
429 ADD_TO_ZONE(zone, newmem);
430 if (from_zone_map(newmem, elem_size))
431 zone_page_alloc(newmem, elem_size);
432 zone->count++; /* compensate for ADD_TO_ZONE */
433 size -= elem_size;
434 newmem += elem_size;
435 zone->cur_size += elem_size;
436 }
437 unlock_zone(zone);
438 }
439
440 /*
441 * Contiguous space allocator for non-paged zones. Allocates "size" amount
442 * of memory from zone_map.
443 */
444
445 kern_return_t
446 zget_space(
447 vm_offset_t size,
448 vm_offset_t *result)
449 {
450 vm_offset_t new_space = 0;
451 vm_size_t space_to_add = 0;
452
453 simple_lock(&zget_space_lock);
454 while ((zalloc_next_space + size) > zalloc_end_of_space) {
455 /*
456 * Add at least one page to allocation area.
457 */
458
459 space_to_add = round_page(size);
460
461 if (new_space == 0) {
462 kern_return_t retval;
463 /*
464 * Memory cannot be wired down while holding
465 * any locks that the pageout daemon might
466 * need to free up pages. [Making the zget_space
467 * lock a complex lock does not help in this
468 * regard.]
469 *
470 * Unlock and allocate memory. Because several
471 * threads might try to do this at once, don't
472 * use the memory before checking for available
473 * space again.
474 */
475
476 simple_unlock(&zget_space_lock);
477
478 retval = kernel_memory_allocate(zone_map, &new_space,
479 space_to_add, 0, KMA_KOBJECT|KMA_NOPAGEWAIT);
480 if (retval != KERN_SUCCESS)
481 return(retval);
482 zone_page_init(new_space, space_to_add,
483 ZONE_PAGE_USED);
484 simple_lock(&zget_space_lock);
485 continue;
486 }
487
488
489 /*
490 * Memory was allocated in a previous iteration.
491 *
492 * Check whether the new region is contiguous
493 * with the old one.
494 */
495
496 if (new_space != zalloc_end_of_space) {
497 /*
498 * Throw away the remainder of the
499 * old space, and start a new one.
500 */
501 zalloc_wasted_space +=
502 zalloc_end_of_space - zalloc_next_space;
503 zalloc_next_space = new_space;
504 }
505
506 zalloc_end_of_space = new_space + space_to_add;
507
508 new_space = 0;
509 }
510 *result = zalloc_next_space;
511 zalloc_next_space += size;
512 simple_unlock(&zget_space_lock);
513
514 if (new_space != 0)
515 kmem_free(zone_map, new_space, space_to_add);
516
517 return(KERN_SUCCESS);
518 }
519
520
521 /*
522 * Steal memory for the zone package. Called from
523 * vm_page_bootstrap().
524 */
525 void
526 zone_steal_memory(void)
527 {
528 zdata_size = round_page(128*sizeof(struct zone));
529 zdata = (vm_offset_t)((char *)pmap_steal_memory(zdata_size) - (char *)0);
530 }
531
532
533 /*
534 * Fill a zone with enough memory to contain at least nelem elements.
535 * Memory is obtained with kmem_alloc_wired from the kernel_map.
536 * Return the number of elements actually put into the zone, which may
537 * be more than the caller asked for since the memory allocation is
538 * rounded up to a full page.
539 */
540 int
541 zfill(
542 zone_t zone,
543 int nelem)
544 {
545 kern_return_t kr;
546 vm_size_t size;
547 vm_offset_t memory;
548 int nalloc;
549
550 assert(nelem > 0);
551 if (nelem <= 0)
552 return 0;
553 size = nelem * zone->elem_size;
554 size = round_page(size);
555 kr = kmem_alloc_wired(kernel_map, &memory, size);
556 if (kr != KERN_SUCCESS)
557 return 0;
558
559 zone_change(zone, Z_FOREIGN, TRUE);
560 zcram(zone, (void *)memory, size);
561 nalloc = size / zone->elem_size;
562 assert(nalloc >= nelem);
563
564 return nalloc;
565 }
566
567 /*
568 * Initialize the "zone of zones" which uses fixed memory allocated
569 * earlier in memory initialization. zone_bootstrap is called
570 * before zone_init.
571 */
572 void
573 zone_bootstrap(void)
574 {
575 vm_size_t zone_zone_size;
576 vm_offset_t zone_zone_space;
577
578 simple_lock_init(&all_zones_lock, 0);
579
580 first_zone = ZONE_NULL;
581 last_zone = &first_zone;
582 num_zones = 0;
583
584 simple_lock_init(&zget_space_lock, 0);
585 zalloc_next_space = zdata;
586 zalloc_end_of_space = zdata + zdata_size;
587 zalloc_wasted_space = 0;
588
589 /* assertion: nobody else called zinit before us */
590 assert(zone_zone == ZONE_NULL);
591 zone_zone = zinit(sizeof(struct zone), 128 * sizeof(struct zone),
592 sizeof(struct zone), "zones");
593 zone_change(zone_zone, Z_COLLECT, FALSE);
594 zone_zone_size = zalloc_end_of_space - zalloc_next_space;
595 zget_space(zone_zone_size, &zone_zone_space);
596 zcram(zone_zone, (void *)zone_zone_space, zone_zone_size);
597 }
598
599 void
600 zone_init(
601 vm_size_t max_zonemap_size)
602 {
603 kern_return_t retval;
604 vm_offset_t zone_min;
605 vm_offset_t zone_max;
606 vm_size_t zone_table_size;
607
608 retval = kmem_suballoc(kernel_map, &zone_min, max_zonemap_size,
609 FALSE, VM_FLAGS_ANYWHERE, &zone_map);
610
611 if (retval != KERN_SUCCESS)
612 panic("zone_init: kmem_suballoc failed");
613 zone_max = zone_min + round_page(max_zonemap_size);
614 /*
615 * Setup garbage collection information:
616 */
617 zone_table_size = atop_32(zone_max - zone_min) *
618 sizeof(struct zone_page_table_entry);
619 if (kmem_alloc_wired(zone_map, (vm_offset_t *) &zone_page_table,
620 zone_table_size) != KERN_SUCCESS)
621 panic("zone_init");
622 zone_min = (vm_offset_t)zone_page_table + round_page(zone_table_size);
623 zone_pages = atop_32(zone_max - zone_min);
624 zone_map_min_address = zone_min;
625 zone_map_max_address = zone_max;
626 mutex_init(&zone_gc_lock, 0);
627 zone_page_init(zone_min, zone_max - zone_min, ZONE_PAGE_UNUSED);
628 }
629
630
631 /*
632 * zalloc returns an element from the specified zone.
633 */
634 void *
635 zalloc_canblock(
636 register zone_t zone,
637 boolean_t canblock)
638 {
639 vm_offset_t addr;
640 kern_return_t retval;
641
642 assert(zone != ZONE_NULL);
643
644 lock_zone(zone);
645
646 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
647
648 while ((addr == 0) && canblock && (zone->doing_gc)) {
649 zone->waiting = TRUE;
650 zone_sleep(zone);
651 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
652 }
653
654 while ((addr == 0) && canblock) {
655 /*
656 * If nothing was there, try to get more
657 */
658 if (zone->doing_alloc) {
659 /*
660 * Someone is allocating memory for this zone.
661 * Wait for it to show up, then try again.
662 */
663 zone->waiting = TRUE;
664 zone_sleep(zone);
665 }
666 else {
667 if ((zone->cur_size + zone->elem_size) >
668 zone->max_size) {
669 if (zone->exhaustible)
670 break;
671 if (zone->expandable) {
672 /*
673 * We're willing to overflow certain
674 * zones, but not without complaining.
675 *
676 * This is best used in conjunction
677 * with the collectable flag. What we
678 * want is an assurance we can get the
679 * memory back, assuming there's no
680 * leak.
681 */
682 zone->max_size += (zone->max_size >> 1);
683 } else {
684 unlock_zone(zone);
685
686 panic("zalloc: zone \"%s\" empty.", zone->zone_name);
687 }
688 }
689 zone->doing_alloc = TRUE;
690 unlock_zone(zone);
691
692 if (zone->collectable) {
693 vm_offset_t space;
694 vm_size_t alloc_size;
695 boolean_t retry = FALSE;
696
697 for (;;) {
698
699 if (vm_pool_low() || retry == TRUE)
700 alloc_size =
701 round_page(zone->elem_size);
702 else
703 alloc_size = zone->alloc_size;
704
705 retval = kernel_memory_allocate(zone_map,
706 &space, alloc_size, 0,
707 KMA_KOBJECT|KMA_NOPAGEWAIT);
708 if (retval == KERN_SUCCESS) {
709 zone_page_init(space, alloc_size,
710 ZONE_PAGE_USED);
711 zcram(zone, (void *)space, alloc_size);
712
713 break;
714 } else if (retval != KERN_RESOURCE_SHORTAGE) {
715 /* would like to cause a zone_gc() */
716 if (retry == TRUE)
717 panic("zalloc: \"%s\" (%d elements) retry fail %d", zone->zone_name, zone->count, retval);
718 retry = TRUE;
719 } else {
720 break;
721 }
722 }
723 lock_zone(zone);
724 zone->doing_alloc = FALSE;
725 if (zone->waiting) {
726 zone->waiting = FALSE;
727 zone_wakeup(zone);
728 }
729 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
730 if (addr == 0 &&
731 retval == KERN_RESOURCE_SHORTAGE) {
732 unlock_zone(zone);
733
734 VM_PAGE_WAIT();
735 lock_zone(zone);
736 }
737 } else {
738 vm_offset_t space;
739 retval = zget_space(zone->elem_size, &space);
740
741 lock_zone(zone);
742 zone->doing_alloc = FALSE;
743 if (zone->waiting) {
744 zone->waiting = FALSE;
745 thread_wakeup((event_t)zone);
746 }
747 if (retval == KERN_SUCCESS) {
748 zone->count++;
749 zone->cur_size += zone->elem_size;
750 #if ZONE_DEBUG
751 if (zone_debug_enabled(zone)) {
752 enqueue_tail(&zone->active_zones, (queue_entry_t)space);
753 }
754 #endif
755 unlock_zone(zone);
756 zone_page_alloc(space, zone->elem_size);
757 #if ZONE_DEBUG
758 if (zone_debug_enabled(zone))
759 space += ZONE_DEBUG_OFFSET;
760 #endif
761 return((void *)space);
762 }
763 if (retval == KERN_RESOURCE_SHORTAGE) {
764 unlock_zone(zone);
765
766 VM_PAGE_WAIT();
767 lock_zone(zone);
768 } else {
769 panic("zalloc: \"%s\" (%d elements) zget_space returned %d", zone->zone_name, zone->count, retval);
770 }
771 }
772 }
773 if (addr == 0)
774 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
775 }
776
777 if ((addr == 0) && !canblock && (zone->async_pending == FALSE) && (!vm_pool_low())) {
778 zone->async_pending = TRUE;
779 unlock_zone(zone);
780 thread_call_enter(&zone->call_async_alloc);
781 lock_zone(zone);
782 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
783 }
784
785 #if ZONE_DEBUG
786 if (addr && zone_debug_enabled(zone)) {
787 enqueue_tail(&zone->active_zones, (queue_entry_t)addr);
788 addr += ZONE_DEBUG_OFFSET;
789 }
790 #endif
791
792 unlock_zone(zone);
793
794 return((void *)addr);
795 }
796
797
798 void *
799 zalloc(
800 register zone_t zone)
801 {
802 return( zalloc_canblock(zone, TRUE) );
803 }
804
805 void *
806 zalloc_noblock(
807 register zone_t zone)
808 {
809 return( zalloc_canblock(zone, FALSE) );
810 }
811
812 void
813 zalloc_async(
814 thread_call_param_t p0,
815 __unused thread_call_param_t p1)
816 {
817 void *elt;
818
819 elt = zalloc_canblock((zone_t)p0, TRUE);
820 zfree((zone_t)p0, elt);
821 lock_zone(((zone_t)p0));
822 ((zone_t)p0)->async_pending = FALSE;
823 unlock_zone(((zone_t)p0));
824 }
825
826
827 /*
828 * zget returns an element from the specified zone
829 * and immediately returns nothing if there is nothing there.
830 *
831 * This form should be used when you can not block (like when
832 * processing an interrupt).
833 */
834 void *
835 zget(
836 register zone_t zone)
837 {
838 register vm_offset_t addr;
839
840 assert( zone != ZONE_NULL );
841
842 if (!lock_try_zone(zone))
843 return NULL;
844
845 REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
846 #if ZONE_DEBUG
847 if (addr && zone_debug_enabled(zone)) {
848 enqueue_tail(&zone->active_zones, (queue_entry_t)addr);
849 addr += ZONE_DEBUG_OFFSET;
850 }
851 #endif /* ZONE_DEBUG */
852 unlock_zone(zone);
853
854 return((void *) addr);
855 }
856
857 /* Keep this FALSE by default. Large memory machine run orders of magnitude
858 slower in debug mode when true. Use debugger to enable if needed */
859 /* static */ boolean_t zone_check = FALSE;
860
861 static zone_t zone_last_bogus_zone = ZONE_NULL;
862 static vm_offset_t zone_last_bogus_elem = 0;
863
864 void
865 zfree(
866 register zone_t zone,
867 void *addr)
868 {
869 vm_offset_t elem = (vm_offset_t) addr;
870
871 #if MACH_ASSERT
872 /* Basic sanity checks */
873 if (zone == ZONE_NULL || elem == (vm_offset_t)0)
874 panic("zfree: NULL");
875 /* zone_gc assumes zones are never freed */
876 if (zone == zone_zone)
877 panic("zfree: freeing to zone_zone breaks zone_gc!");
878 #endif
879
880 if (zone->collectable && !zone->allows_foreign &&
881 !from_zone_map(elem, zone->elem_size)) {
882 #if MACH_ASSERT
883 panic("zfree: non-allocated memory in collectable zone!");
884 #endif
885 zone_last_bogus_zone = zone;
886 zone_last_bogus_elem = elem;
887 return;
888 }
889
890 lock_zone(zone);
891 #if ZONE_DEBUG
892 if (zone_debug_enabled(zone)) {
893 queue_t tmp_elem;
894
895 elem -= ZONE_DEBUG_OFFSET;
896 if (zone_check) {
897 /* check the zone's consistency */
898
899 for (tmp_elem = queue_first(&zone->active_zones);
900 !queue_end(tmp_elem, &zone->active_zones);
901 tmp_elem = queue_next(tmp_elem))
902 if (elem == (vm_offset_t)tmp_elem)
903 break;
904 if (elem != (vm_offset_t)tmp_elem)
905 panic("zfree()ing element from wrong zone");
906 }
907 remqueue(&zone->active_zones, (queue_t) elem);
908 }
909 #endif /* ZONE_DEBUG */
910 if (zone_check) {
911 vm_offset_t this;
912
913 /* check the zone's consistency */
914
915 for (this = zone->free_elements;
916 this != 0;
917 this = * (vm_offset_t *) this)
918 if (!pmap_kernel_va(this) || this == elem)
919 panic("zfree");
920 }
921 ADD_TO_ZONE(zone, elem);
922
923 /*
924 * If elements have one or more pages, and memory is low,
925 * request to run the garbage collection in the zone the next
926 * time the pageout thread runs.
927 */
928 if (zone->elem_size >= PAGE_SIZE &&
929 vm_pool_low()){
930 zone_gc_forced = TRUE;
931 }
932 unlock_zone(zone);
933 }
934
935
936 /* Change a zone's flags.
937 * This routine must be called immediately after zinit.
938 */
939 void
940 zone_change(
941 zone_t zone,
942 unsigned int item,
943 boolean_t value)
944 {
945 assert( zone != ZONE_NULL );
946 assert( value == TRUE || value == FALSE );
947
948 switch(item){
949 case Z_EXHAUST:
950 zone->exhaustible = value;
951 break;
952 case Z_COLLECT:
953 zone->collectable = value;
954 break;
955 case Z_EXPAND:
956 zone->expandable = value;
957 break;
958 case Z_FOREIGN:
959 zone->allows_foreign = value;
960 break;
961 #if MACH_ASSERT
962 default:
963 panic("Zone_change: Wrong Item Type!");
964 /* break; */
965 #endif
966 }
967 lock_zone_init(zone);
968 }
969
970 /*
971 * Return the expected number of free elements in the zone.
972 * This calculation will be incorrect if items are zfree'd that
973 * were never zalloc'd/zget'd. The correct way to stuff memory
974 * into a zone is by zcram.
975 */
976
977 integer_t
978 zone_free_count(zone_t zone)
979 {
980 integer_t free_count;
981
982 lock_zone(zone);
983 free_count = zone->cur_size/zone->elem_size - zone->count;
984 unlock_zone(zone);
985
986 assert(free_count >= 0);
987
988 return(free_count);
989 }
990
991 /*
992 * zprealloc preallocates wired memory, exanding the specified
993 * zone to the specified size
994 */
995 void
996 zprealloc(
997 zone_t zone,
998 vm_size_t size)
999 {
1000 vm_offset_t addr;
1001
1002 if (size != 0) {
1003 if (kmem_alloc_wired(zone_map, &addr, size) != KERN_SUCCESS)
1004 panic("zprealloc");
1005 zone_page_init(addr, size, ZONE_PAGE_USED);
1006 zcram(zone, (void *)addr, size);
1007 }
1008 }
1009
1010 /*
1011 * Zone garbage collection subroutines
1012 */
1013
1014 boolean_t
1015 zone_page_collectable(
1016 vm_offset_t addr,
1017 vm_size_t size)
1018 {
1019 struct zone_page_table_entry *zp;
1020 natural_t i, j;
1021
1022 #if MACH_ASSERT
1023 if (!from_zone_map(addr, size))
1024 panic("zone_page_collectable");
1025 #endif
1026
1027 i = atop_32(addr-zone_map_min_address);
1028 j = atop_32((addr+size-1) - zone_map_min_address);
1029
1030 for (zp = zone_page_table + i; i <= j; zp++, i++)
1031 if (zp->collect_count == zp->alloc_count)
1032 return (TRUE);
1033
1034 return (FALSE);
1035 }
1036
1037 void
1038 zone_page_keep(
1039 vm_offset_t addr,
1040 vm_size_t size)
1041 {
1042 struct zone_page_table_entry *zp;
1043 natural_t i, j;
1044
1045 #if MACH_ASSERT
1046 if (!from_zone_map(addr, size))
1047 panic("zone_page_keep");
1048 #endif
1049
1050 i = atop_32(addr-zone_map_min_address);
1051 j = atop_32((addr+size-1) - zone_map_min_address);
1052
1053 for (zp = zone_page_table + i; i <= j; zp++, i++)
1054 zp->collect_count = 0;
1055 }
1056
1057 void
1058 zone_page_collect(
1059 vm_offset_t addr,
1060 vm_size_t size)
1061 {
1062 struct zone_page_table_entry *zp;
1063 natural_t i, j;
1064
1065 #if MACH_ASSERT
1066 if (!from_zone_map(addr, size))
1067 panic("zone_page_collect");
1068 #endif
1069
1070 i = atop_32(addr-zone_map_min_address);
1071 j = atop_32((addr+size-1) - zone_map_min_address);
1072
1073 for (zp = zone_page_table + i; i <= j; zp++, i++)
1074 ++zp->collect_count;
1075 }
1076
1077 void
1078 zone_page_init(
1079 vm_offset_t addr,
1080 vm_size_t size,
1081 int value)
1082 {
1083 struct zone_page_table_entry *zp;
1084 natural_t i, j;
1085
1086 #if MACH_ASSERT
1087 if (!from_zone_map(addr, size))
1088 panic("zone_page_init");
1089 #endif
1090
1091 i = atop_32(addr-zone_map_min_address);
1092 j = atop_32((addr+size-1) - zone_map_min_address);
1093
1094 for (zp = zone_page_table + i; i <= j; zp++, i++) {
1095 zp->alloc_count = value;
1096 zp->collect_count = 0;
1097 }
1098 }
1099
1100 void
1101 zone_page_alloc(
1102 vm_offset_t addr,
1103 vm_size_t size)
1104 {
1105 struct zone_page_table_entry *zp;
1106 natural_t i, j;
1107
1108 #if MACH_ASSERT
1109 if (!from_zone_map(addr, size))
1110 panic("zone_page_alloc");
1111 #endif
1112
1113 i = atop_32(addr-zone_map_min_address);
1114 j = atop_32((addr+size-1) - zone_map_min_address);
1115
1116 for (zp = zone_page_table + i; i <= j; zp++, i++) {
1117 /*
1118 * Set alloc_count to (ZONE_PAGE_USED + 1) if
1119 * it was previously set to ZONE_PAGE_UNUSED.
1120 */
1121 if (zp->alloc_count == ZONE_PAGE_UNUSED)
1122 zp->alloc_count = 1;
1123 else
1124 ++zp->alloc_count;
1125 }
1126 }
1127
1128 void
1129 zone_page_free_element(
1130 struct zone_page_table_entry **free_pages,
1131 vm_offset_t addr,
1132 vm_size_t size)
1133 {
1134 struct zone_page_table_entry *zp;
1135 natural_t i, j;
1136
1137 #if MACH_ASSERT
1138 if (!from_zone_map(addr, size))
1139 panic("zone_page_free_element");
1140 #endif
1141
1142 i = atop_32(addr-zone_map_min_address);
1143 j = atop_32((addr+size-1) - zone_map_min_address);
1144
1145 for (zp = zone_page_table + i; i <= j; zp++, i++) {
1146 if (zp->collect_count > 0)
1147 --zp->collect_count;
1148 if (--zp->alloc_count == 0) {
1149 zp->alloc_count = ZONE_PAGE_UNUSED;
1150 zp->collect_count = 0;
1151
1152 zp->link = *free_pages;
1153 *free_pages = zp;
1154 }
1155 }
1156 }
1157
1158
1159 /* This is used for walking through a zone's free element list.
1160 */
1161 struct zone_free_element {
1162 struct zone_free_element * next;
1163 };
1164
1165 struct {
1166 uint32_t pgs_freed;
1167
1168 uint32_t elems_collected,
1169 elems_freed,
1170 elems_kept;
1171 } zgc_stats;
1172
1173 /* Zone garbage collection
1174 *
1175 * zone_gc will walk through all the free elements in all the
1176 * zones that are marked collectable looking for reclaimable
1177 * pages. zone_gc is called by consider_zone_gc when the system
1178 * begins to run out of memory.
1179 */
1180 void
1181 zone_gc(void)
1182 {
1183 unsigned int max_zones;
1184 zone_t z;
1185 unsigned int i;
1186 struct zone_page_table_entry *zp, *zone_free_pages;
1187
1188 mutex_lock(&zone_gc_lock);
1189
1190 simple_lock(&all_zones_lock);
1191 max_zones = num_zones;
1192 z = first_zone;
1193 simple_unlock(&all_zones_lock);
1194
1195 #if MACH_ASSERT
1196 for (i = 0; i < zone_pages; i++)
1197 assert(zone_page_table[i].collect_count == 0);
1198 #endif /* MACH_ASSERT */
1199
1200 zone_free_pages = NULL;
1201
1202 for (i = 0; i < max_zones; i++, z = z->next_zone) {
1203 unsigned int n, m;
1204 vm_size_t elt_size, size_freed;
1205 struct zone_free_element *elt, *base_elt, *base_prev, *prev, *scan, *keep, *tail;
1206
1207 assert(z != ZONE_NULL);
1208
1209 if (!z->collectable)
1210 continue;
1211
1212 lock_zone(z);
1213
1214 elt_size = z->elem_size;
1215
1216 /*
1217 * Do a quick feasability check before we scan the zone:
1218 * skip unless there is likelihood of getting pages back
1219 * (i.e we need a whole allocation block's worth of free
1220 * elements before we can garbage collect) and
1221 * the zone has more than 10 percent of it's elements free
1222 */
1223 if (((z->cur_size - z->count * elt_size) <= (2 * z->alloc_size)) ||
1224 ((z->cur_size - z->count * elt_size) <= (z->cur_size / 10))) {
1225 unlock_zone(z);
1226 continue;
1227 }
1228
1229 z->doing_gc = TRUE;
1230
1231 /*
1232 * Snatch all of the free elements away from the zone.
1233 */
1234
1235 scan = (void *)z->free_elements;
1236 z->free_elements = 0;
1237
1238 unlock_zone(z);
1239
1240 /*
1241 * Pass 1:
1242 *
1243 * Determine which elements we can attempt to collect
1244 * and count them up in the page table. Foreign elements
1245 * are returned to the zone.
1246 */
1247
1248 prev = (void *)&scan;
1249 elt = scan;
1250 n = 0; tail = keep = NULL;
1251 while (elt != NULL) {
1252 if (from_zone_map(elt, elt_size)) {
1253 zone_page_collect((vm_offset_t)elt, elt_size);
1254
1255 prev = elt;
1256 elt = elt->next;
1257
1258 ++zgc_stats.elems_collected;
1259 }
1260 else {
1261 if (keep == NULL)
1262 keep = tail = elt;
1263 else
1264 tail = tail->next = elt;
1265
1266 elt = prev->next = elt->next;
1267 tail->next = NULL;
1268 }
1269
1270 /*
1271 * Dribble back the elements we are keeping.
1272 */
1273
1274 if (++n >= 50) {
1275 if (z->waiting == TRUE) {
1276 lock_zone(z);
1277
1278 if (keep != NULL) {
1279 tail->next = (void *)z->free_elements;
1280 z->free_elements = (vm_offset_t) keep;
1281 tail = keep = NULL;
1282 } else {
1283 m =0;
1284 base_elt = elt;
1285 base_prev = prev;
1286 while ((elt != NULL) && (++m < 50)) {
1287 prev = elt;
1288 elt = elt->next;
1289 }
1290 if (m !=0 ) {
1291 prev->next = (void *)z->free_elements;
1292 z->free_elements = (vm_offset_t) base_elt;
1293 base_prev->next = elt;
1294 prev = base_prev;
1295 }
1296 }
1297
1298 if (z->waiting) {
1299 z->waiting = FALSE;
1300 zone_wakeup(z);
1301 }
1302
1303 unlock_zone(z);
1304 }
1305 n =0;
1306 }
1307 }
1308
1309 /*
1310 * Return any remaining elements.
1311 */
1312
1313 if (keep != NULL) {
1314 lock_zone(z);
1315
1316 tail->next = (void *)z->free_elements;
1317 z->free_elements = (vm_offset_t) keep;
1318
1319 unlock_zone(z);
1320 }
1321
1322 /*
1323 * Pass 2:
1324 *
1325 * Determine which pages we can reclaim and
1326 * free those elements.
1327 */
1328
1329 size_freed = 0;
1330 prev = (void *)&scan;
1331 elt = scan;
1332 n = 0; tail = keep = NULL;
1333 while (elt != NULL) {
1334 if (zone_page_collectable((vm_offset_t)elt, elt_size)) {
1335 size_freed += elt_size;
1336 zone_page_free_element(&zone_free_pages,
1337 (vm_offset_t)elt, elt_size);
1338
1339 elt = prev->next = elt->next;
1340
1341 ++zgc_stats.elems_freed;
1342 }
1343 else {
1344 zone_page_keep((vm_offset_t)elt, elt_size);
1345
1346 if (keep == NULL)
1347 keep = tail = elt;
1348 else
1349 tail = tail->next = elt;
1350
1351 elt = prev->next = elt->next;
1352 tail->next = NULL;
1353
1354 ++zgc_stats.elems_kept;
1355 }
1356
1357 /*
1358 * Dribble back the elements we are keeping,
1359 * and update the zone size info.
1360 */
1361
1362 if (++n >= 50) {
1363 lock_zone(z);
1364
1365 z->cur_size -= size_freed;
1366 size_freed = 0;
1367
1368 if (keep != NULL) {
1369 tail->next = (void *)z->free_elements;
1370 z->free_elements = (vm_offset_t) keep;
1371 }
1372
1373 if (z->waiting) {
1374 z->waiting = FALSE;
1375 zone_wakeup(z);
1376 }
1377
1378 unlock_zone(z);
1379
1380 n = 0; tail = keep = NULL;
1381 }
1382 }
1383
1384 /*
1385 * Return any remaining elements, and update
1386 * the zone size info.
1387 */
1388
1389 lock_zone(z);
1390
1391 if (size_freed > 0 || keep != NULL) {
1392
1393 z->cur_size -= size_freed;
1394
1395 if (keep != NULL) {
1396 tail->next = (void *)z->free_elements;
1397 z->free_elements = (vm_offset_t) keep;
1398 }
1399
1400 }
1401
1402 z->doing_gc = FALSE;
1403 if (z->waiting) {
1404 z->waiting = FALSE;
1405 zone_wakeup(z);
1406 }
1407 unlock_zone(z);
1408 }
1409
1410 /*
1411 * Reclaim the pages we are freeing.
1412 */
1413
1414 while ((zp = zone_free_pages) != NULL) {
1415 zone_free_pages = zp->link;
1416 kmem_free(zone_map, zone_map_min_address + PAGE_SIZE *
1417 (zp - zone_page_table), PAGE_SIZE);
1418 ++zgc_stats.pgs_freed;
1419 }
1420
1421 mutex_unlock(&zone_gc_lock);
1422 }
1423
1424 /*
1425 * consider_zone_gc:
1426 *
1427 * Called by the pageout daemon when the system needs more free pages.
1428 */
1429
1430 void
1431 consider_zone_gc(void)
1432 {
1433 /*
1434 * By default, don't attempt zone GC more frequently
1435 * than once / 1 minutes.
1436 */
1437
1438 if (zone_gc_max_rate == 0)
1439 zone_gc_max_rate = (60 << SCHED_TICK_SHIFT) + 1;
1440
1441 if (zone_gc_allowed &&
1442 ((sched_tick > (zone_gc_last_tick + zone_gc_max_rate)) ||
1443 zone_gc_forced)) {
1444 zone_gc_forced = FALSE;
1445 zone_gc_last_tick = sched_tick;
1446 zone_gc();
1447 }
1448 }
1449
1450
1451 kern_return_t
1452 host_zone_info(
1453 host_t host,
1454 zone_name_array_t *namesp,
1455 mach_msg_type_number_t *namesCntp,
1456 zone_info_array_t *infop,
1457 mach_msg_type_number_t *infoCntp)
1458 {
1459 zone_name_t *names;
1460 vm_offset_t names_addr;
1461 vm_size_t names_size;
1462 zone_info_t *info;
1463 vm_offset_t info_addr;
1464 vm_size_t info_size;
1465 unsigned int max_zones, i;
1466 zone_t z;
1467 zone_name_t *zn;
1468 zone_info_t *zi;
1469 kern_return_t kr;
1470
1471 if (host == HOST_NULL)
1472 return KERN_INVALID_HOST;
1473
1474 /*
1475 * We assume that zones aren't freed once allocated.
1476 * We won't pick up any zones that are allocated later.
1477 */
1478
1479 simple_lock(&all_zones_lock);
1480 #ifdef ppc
1481 max_zones = num_zones + 4;
1482 #else
1483 max_zones = num_zones + 3; /* ATN: count the number below!! */
1484 #endif
1485 z = first_zone;
1486 simple_unlock(&all_zones_lock);
1487
1488 if (max_zones <= *namesCntp) {
1489 /* use in-line memory */
1490 names_size = *namesCntp * sizeof *names;
1491 names = *namesp;
1492 } else {
1493 names_size = round_page(max_zones * sizeof *names);
1494 kr = kmem_alloc_pageable(ipc_kernel_map,
1495 &names_addr, names_size);
1496 if (kr != KERN_SUCCESS)
1497 return kr;
1498 names = (zone_name_t *) names_addr;
1499 }
1500
1501 if (max_zones <= *infoCntp) {
1502 /* use in-line memory */
1503 info_size = *infoCntp * sizeof *info;
1504 info = *infop;
1505 } else {
1506 info_size = round_page(max_zones * sizeof *info);
1507 kr = kmem_alloc_pageable(ipc_kernel_map,
1508 &info_addr, info_size);
1509 if (kr != KERN_SUCCESS) {
1510 if (names != *namesp)
1511 kmem_free(ipc_kernel_map,
1512 names_addr, names_size);
1513 return kr;
1514 }
1515
1516 info = (zone_info_t *) info_addr;
1517 }
1518 zn = &names[0];
1519 zi = &info[0];
1520
1521 for (i = 0; i < num_zones; i++) {
1522 struct zone zcopy;
1523
1524 assert(z != ZONE_NULL);
1525
1526 lock_zone(z);
1527 zcopy = *z;
1528 unlock_zone(z);
1529
1530 simple_lock(&all_zones_lock);
1531 z = z->next_zone;
1532 simple_unlock(&all_zones_lock);
1533
1534 /* assuming here the name data is static */
1535 (void) strncpy(zn->zn_name, zcopy.zone_name,
1536 sizeof zn->zn_name);
1537
1538 zi->zi_count = zcopy.count;
1539 zi->zi_cur_size = zcopy.cur_size;
1540 zi->zi_max_size = zcopy.max_size;
1541 zi->zi_elem_size = zcopy.elem_size;
1542 zi->zi_alloc_size = zcopy.alloc_size;
1543 zi->zi_exhaustible = zcopy.exhaustible;
1544 zi->zi_collectable = zcopy.collectable;
1545
1546 zn++;
1547 zi++;
1548 }
1549 strcpy(zn->zn_name, "kernel_stacks");
1550 stack_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
1551 &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
1552 zn++;
1553 zi++;
1554 #ifdef ppc
1555 strcpy(zn->zn_name, "save_areas");
1556 save_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
1557 &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
1558 zn++;
1559 zi++;
1560
1561 strcpy(zn->zn_name, "pmap_mappings");
1562 mapping_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
1563 &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
1564 zn++;
1565 zi++;
1566 #endif
1567
1568 #ifdef i386
1569 strcpy(zn->zn_name, "page_tables");
1570 pt_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
1571 &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
1572 zn++;
1573 zi++;
1574 #endif
1575
1576 strcpy(zn->zn_name, "kalloc.large");
1577 kalloc_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
1578 &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
1579
1580 if (names != *namesp) {
1581 vm_size_t used;
1582 vm_map_copy_t copy;
1583
1584 used = max_zones * sizeof *names;
1585
1586 if (used != names_size)
1587 bzero((char *) (names_addr + used), names_size - used);
1588
1589 kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)names_addr,
1590 (vm_map_size_t)names_size, TRUE, &copy);
1591 assert(kr == KERN_SUCCESS);
1592
1593 *namesp = (zone_name_t *) copy;
1594 }
1595 *namesCntp = max_zones;
1596
1597 if (info != *infop) {
1598 vm_size_t used;
1599 vm_map_copy_t copy;
1600
1601 used = max_zones * sizeof *info;
1602
1603 if (used != info_size)
1604 bzero((char *) (info_addr + used), info_size - used);
1605
1606 kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)info_addr,
1607 (vm_map_size_t)info_size, TRUE, &copy);
1608 assert(kr == KERN_SUCCESS);
1609
1610 *infop = (zone_info_t *) copy;
1611 }
1612 *infoCntp = max_zones;
1613
1614 return KERN_SUCCESS;
1615 }
1616
1617 #if MACH_KDB
1618 #include <ddb/db_command.h>
1619 #include <ddb/db_output.h>
1620 #include <kern/kern_print.h>
1621
1622 const char *zone_labels =
1623 "ENTRY COUNT TOT_SZ MAX_SZ ELT_SZ ALLOC_SZ NAME";
1624
1625 /* Forwards */
1626 void db_print_zone(
1627 zone_t addr);
1628
1629 #if ZONE_DEBUG
1630 void db_zone_check_active(
1631 zone_t zone);
1632 void db_zone_print_active(
1633 zone_t zone);
1634 #endif /* ZONE_DEBUG */
1635 void db_zone_print_free(
1636 zone_t zone);
1637 void
1638 db_print_zone(
1639 zone_t addr)
1640 {
1641 struct zone zcopy;
1642
1643 zcopy = *addr;
1644
1645 db_printf("%8x %8x %8x %8x %6x %8x %s ",
1646 addr, zcopy.count, zcopy.cur_size,
1647 zcopy.max_size, zcopy.elem_size,
1648 zcopy.alloc_size, zcopy.zone_name);
1649 if (zcopy.exhaustible)
1650 db_printf("H");
1651 if (zcopy.collectable)
1652 db_printf("C");
1653 if (zcopy.expandable)
1654 db_printf("X");
1655 db_printf("\n");
1656 }
1657
1658 /*ARGSUSED*/
1659 void
1660 db_show_one_zone(
1661 db_expr_t addr,
1662 int have_addr,
1663 __unused db_expr_t count,
1664 __unused char * modif)
1665 {
1666 struct zone *z = (zone_t)((char *)0 + addr);
1667
1668 if (z == ZONE_NULL || !have_addr){
1669 db_error("No Zone\n");
1670 /*NOTREACHED*/
1671 }
1672
1673 db_printf("%s\n", zone_labels);
1674 db_print_zone(z);
1675 }
1676
1677 /*ARGSUSED*/
1678 void
1679 db_show_all_zones(
1680 __unused db_expr_t addr,
1681 int have_addr,
1682 db_expr_t count,
1683 __unused char * modif)
1684 {
1685 zone_t z;
1686 unsigned total = 0;
1687
1688 /*
1689 * Don't risk hanging by unconditionally locking,
1690 * risk of incoherent data is small (zones aren't freed).
1691 */
1692 have_addr = simple_lock_try(&all_zones_lock);
1693 count = num_zones;
1694 z = first_zone;
1695 if (have_addr) {
1696 simple_unlock(&all_zones_lock);
1697 }
1698
1699 db_printf("%s\n", zone_labels);
1700 for ( ; count > 0; count--) {
1701 if (!z) {
1702 db_error("Mangled Zone List\n");
1703 /*NOTREACHED*/
1704 }
1705 db_print_zone(z);
1706 total += z->cur_size,
1707
1708 have_addr = simple_lock_try(&all_zones_lock);
1709 z = z->next_zone;
1710 if (have_addr) {
1711 simple_unlock(&all_zones_lock);
1712 }
1713 }
1714 db_printf("\nTotal %8x", total);
1715 db_printf("\n\nzone_gc() has reclaimed %d pages\n", zgc_stats.pgs_freed);
1716 }
1717
1718 #if ZONE_DEBUG
1719 void
1720 db_zone_check_active(
1721 zone_t zone)
1722 {
1723 int count = 0;
1724 queue_t tmp_elem;
1725
1726 if (!zone_debug_enabled(zone) || !zone_check)
1727 return;
1728 tmp_elem = queue_first(&zone->active_zones);
1729 while (count < zone->count) {
1730 count++;
1731 if (tmp_elem == 0) {
1732 printf("unexpected zero element, zone=0x%x, count=%d\n",
1733 zone, count);
1734 assert(FALSE);
1735 break;
1736 }
1737 if (queue_end(tmp_elem, &zone->active_zones)) {
1738 printf("unexpected queue_end, zone=0x%x, count=%d\n",
1739 zone, count);
1740 assert(FALSE);
1741 break;
1742 }
1743 tmp_elem = queue_next(tmp_elem);
1744 }
1745 if (!queue_end(tmp_elem, &zone->active_zones)) {
1746 printf("not at queue_end, zone=0x%x, tmp_elem=0x%x\n",
1747 zone, tmp_elem);
1748 assert(FALSE);
1749 }
1750 }
1751
1752 void
1753 db_zone_print_active(
1754 zone_t zone)
1755 {
1756 int count = 0;
1757 queue_t tmp_elem;
1758
1759 if (!zone_debug_enabled(zone)) {
1760 printf("zone 0x%x debug not enabled\n", zone);
1761 return;
1762 }
1763 if (!zone_check) {
1764 printf("zone_check FALSE\n");
1765 return;
1766 }
1767
1768 printf("zone 0x%x, active elements %d\n", zone, zone->count);
1769 printf("active list:\n");
1770 tmp_elem = queue_first(&zone->active_zones);
1771 while (count < zone->count) {
1772 printf(" 0x%x", tmp_elem);
1773 count++;
1774 if ((count % 6) == 0)
1775 printf("\n");
1776 if (tmp_elem == 0) {
1777 printf("\nunexpected zero element, count=%d\n", count);
1778 break;
1779 }
1780 if (queue_end(tmp_elem, &zone->active_zones)) {
1781 printf("\nunexpected queue_end, count=%d\n", count);
1782 break;
1783 }
1784 tmp_elem = queue_next(tmp_elem);
1785 }
1786 if (!queue_end(tmp_elem, &zone->active_zones))
1787 printf("\nnot at queue_end, tmp_elem=0x%x\n", tmp_elem);
1788 else
1789 printf("\n");
1790 }
1791 #endif /* ZONE_DEBUG */
1792
1793 void
1794 db_zone_print_free(
1795 zone_t zone)
1796 {
1797 int count = 0;
1798 int freecount;
1799 vm_offset_t elem;
1800
1801 freecount = zone_free_count(zone);
1802 printf("zone 0x%x, free elements %d\n", zone, freecount);
1803 printf("free list:\n");
1804 elem = zone->free_elements;
1805 while (count < freecount) {
1806 printf(" 0x%x", elem);
1807 count++;
1808 if ((count % 6) == 0)
1809 printf("\n");
1810 if (elem == 0) {
1811 printf("\nunexpected zero element, count=%d\n", count);
1812 break;
1813 }
1814 elem = *((vm_offset_t *)elem);
1815 }
1816 if (elem != 0)
1817 printf("\nnot at end of free list, elem=0x%x\n", elem);
1818 else
1819 printf("\n");
1820 }
1821
1822 #endif /* MACH_KDB */
1823
1824
1825 #if ZONE_DEBUG
1826
1827 /* should we care about locks here ? */
1828
1829 #if MACH_KDB
1830 void *
1831 next_element(
1832 zone_t z,
1833 void *prev)
1834 {
1835 char *elt = (char *)prev;
1836
1837 if (!zone_debug_enabled(z))
1838 return(0);
1839 elt -= ZONE_DEBUG_OFFSET;
1840 elt = (char *) queue_next((queue_t) elt);
1841 if ((queue_t) elt == &z->active_zones)
1842 return(0);
1843 elt += ZONE_DEBUG_OFFSET;
1844 return(elt);
1845 }
1846
1847 void *
1848 first_element(
1849 zone_t z)
1850 {
1851 char *elt;
1852
1853 if (!zone_debug_enabled(z))
1854 return(0);
1855 if (queue_empty(&z->active_zones))
1856 return(0);
1857 elt = (char *)queue_first(&z->active_zones);
1858 elt += ZONE_DEBUG_OFFSET;
1859 return(elt);
1860 }
1861
1862 /*
1863 * Second arg controls how many zone elements are printed:
1864 * 0 => none
1865 * n, n < 0 => all
1866 * n, n > 0 => last n on active list
1867 */
1868 int
1869 zone_count(
1870 zone_t z,
1871 int tail)
1872 {
1873 void *elt;
1874 int count = 0;
1875 boolean_t print = (tail != 0);
1876
1877 if (tail < 0)
1878 tail = z->count;
1879 if (z->count < tail)
1880 tail = 0;
1881 tail = z->count - tail;
1882 for (elt = first_element(z); elt; elt = next_element(z, elt)) {
1883 if (print && tail <= count)
1884 db_printf("%8x\n", elt);
1885 count++;
1886 }
1887 assert(count == z->count);
1888 return(count);
1889 }
1890 #endif /* MACH_KDB */
1891
1892 #define zone_in_use(z) ( z->count || z->free_elements )
1893
1894 void
1895 zone_debug_enable(
1896 zone_t z)
1897 {
1898 if (zone_debug_enabled(z) || zone_in_use(z) ||
1899 z->alloc_size < (z->elem_size + ZONE_DEBUG_OFFSET))
1900 return;
1901 queue_init(&z->active_zones);
1902 z->elem_size += ZONE_DEBUG_OFFSET;
1903 }
1904
1905 void
1906 zone_debug_disable(
1907 zone_t z)
1908 {
1909 if (!zone_debug_enabled(z) || zone_in_use(z))
1910 return;
1911 z->elem_size -= ZONE_DEBUG_OFFSET;
1912 z->active_zones.next = z->active_zones.prev = 0;
1913 }
1914 #endif /* ZONE_DEBUG */
mirror of XNU