]> git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/kalloc.c
projects / apple / xnu.git / blob
? search:


6c4127fa6855742cbaa046b627220a06e88caa36
[apple/xnu.git] / osfmk / kern / kalloc.c
1 /*
2 * Copyright (c) 2000-2020 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: kern/kalloc.c
60 * Author: Avadis Tevanian, Jr.
61 * Date: 1985
62 *
63 * General kernel memory allocator. This allocator is designed
64 * to be used by the kernel to manage dynamic memory fast.
65 */
66
67 #include <mach/boolean.h>
68 #include <mach/sdt.h>
69 #include <mach/machine/vm_types.h>
70 #include <mach/vm_param.h>
71 #include <kern/misc_protos.h>
72 #include <kern/zalloc_internal.h>
73 #include <kern/kalloc.h>
74 #include <kern/ledger.h>
75 #include <kern/backtrace.h>
76 #include <vm/vm_kern.h>
77 #include <vm/vm_object.h>
78 #include <vm/vm_map.h>
79 #include <sys/kdebug.h>
80
81 #include <san/kasan.h>
82 #include <libkern/section_keywords.h>
83
84 /* #define KALLOC_DEBUG 1 */
85
86 #define KALLOC_MAP_SIZE_MIN (16 * 1024 * 1024)
87 #define KALLOC_MAP_SIZE_MAX (128 * 1024 * 1024)
88
89 static SECURITY_READ_ONLY_LATE(vm_offset_t) kalloc_map_min;
90 static SECURITY_READ_ONLY_LATE(vm_offset_t) kalloc_map_max;
91 static SECURITY_READ_ONLY_LATE(vm_size_t) kalloc_max;
92 SECURITY_READ_ONLY_LATE(vm_size_t) kalloc_max_prerounded;
93 /* size of kallocs that can come from kernel map */
94 SECURITY_READ_ONLY_LATE(vm_size_t) kalloc_kernmap_size;
95 SECURITY_READ_ONLY_LATE(vm_map_t) kalloc_map;
96 #if DEBUG || DEVELOPMENT
97 static TUNABLE(bool, kheap_temp_debug, "kheap_temp_debug", false);
98
99 #define KHT_BT_COUNT 14
100 struct kheap_temp_header {
101 queue_chain_t kht_hdr_link;
102 uintptr_t kht_hdr_pcs[KHT_BT_COUNT];
103 };
104 #endif
105
106 /* how many times we couldn't allocate out of kalloc_map and fell back to kernel_map */
107 unsigned long kalloc_fallback_count;
108
109 uint_t kalloc_large_inuse;
110 vm_size_t kalloc_large_total;
111 vm_size_t kalloc_large_max;
112 vm_size_t kalloc_largest_allocated = 0;
113 uint64_t kalloc_large_sum;
114
115 LCK_GRP_DECLARE(kalloc_lck_grp, "kalloc.large");
116 LCK_SPIN_DECLARE(kalloc_lock, &kalloc_lck_grp);
117
118 #define kalloc_spin_lock() lck_spin_lock(&kalloc_lock)
119 #define kalloc_unlock() lck_spin_unlock(&kalloc_lock)
120
121 #pragma mark initialization
122
123 /*
124 * All allocations of size less than kalloc_max are rounded to the next nearest
125 * sized zone. This allocator is built on top of the zone allocator. A zone
126 * is created for each potential size that we are willing to get in small
127 * blocks.
128 *
129 * We assume that kalloc_max is not greater than 64K;
130 *
131 * Note that kalloc_max is somewhat confusingly named. It represents the first
132 * power of two for which no zone exists. kalloc_max_prerounded is the
133 * smallest allocation size, before rounding, for which no zone exists.
134 *
135 * Also if the allocation size is more than kalloc_kernmap_size then allocate
136 * from kernel map rather than kalloc_map.
137 */
138
139 #define KiB(x) (1024 * (x))
140
141 /*
142 * The k_zone_cfg table defines the configuration of zones on various platforms.
143 * The currently defined list of zones and their per-CPU caching behavior are as
144 * follows
145 *
146 * X:zone not present
147 * N:zone present no cpu-caching
148 * Y:zone present with cpu-caching
149 *
150 * Size macOS(64-bit) embedded(32-bit) embedded(64-bit)
151 *-------- ---------------- ---------------- ----------------
152 *
153 * 8 X Y X
154 * 16 Y Y Y
155 * 24 X Y X
156 * 32 Y Y Y
157 * 40 X Y X
158 * 48 Y Y Y
159 * 64 Y Y Y
160 * 72 X Y X
161 * 80 Y X Y
162 * 88 X Y X
163 * 96 Y X Y
164 * 112 X Y X
165 * 128 Y Y Y
166 * 160 Y X Y
167 * 192 Y Y Y
168 * 224 Y X Y
169 * 256 Y Y Y
170 * 288 Y Y Y
171 * 368 Y X Y
172 * 384 X Y X
173 * 400 Y X Y
174 * 440 X Y X
175 * 512 Y Y Y
176 * 576 Y N N
177 * 768 Y N N
178 * 1024 Y Y Y
179 * 1152 N N N
180 * 1280 N N N
181 * 1536 X N X
182 * 1664 N X N
183 * 2048 Y N N
184 * 2128 X N X
185 * 3072 X N X
186 * 4096 Y N N
187 * 6144 N N N
188 * 8192 Y N N
189 * 12288 N X X
190 * 16384 N X N
191 * 32768 X X N
192 *
193 */
194 struct kalloc_zone_cfg {
195 bool kzc_caching;
196 uint32_t kzc_size;
197 const char *kzc_name;
198 };
199 static SECURITY_READ_ONLY_LATE(struct kalloc_zone_cfg) k_zone_cfg[] = {
200 #define KZC_ENTRY(SIZE, caching) { \
201 .kzc_caching = (caching), \
202 .kzc_size = (SIZE), \
203 .kzc_name = "kalloc." #SIZE \
204 }
205
206 #if !defined(XNU_TARGET_OS_OSX)
207
208 #if KALLOC_MINSIZE == 16 && KALLOC_LOG2_MINALIGN == 4
209 /* Zone config for embedded 64-bit platforms */
210 KZC_ENTRY(16, true),
211 KZC_ENTRY(32, true),
212 KZC_ENTRY(48, true),
213 KZC_ENTRY(64, true),
214 KZC_ENTRY(80, true),
215 KZC_ENTRY(96, true),
216 KZC_ENTRY(128, true),
217 KZC_ENTRY(160, true),
218 KZC_ENTRY(192, true),
219 KZC_ENTRY(224, true),
220 KZC_ENTRY(256, true),
221 KZC_ENTRY(288, true),
222 KZC_ENTRY(368, true),
223 KZC_ENTRY(400, true),
224 KZC_ENTRY(512, true),
225 KZC_ENTRY(576, false),
226 KZC_ENTRY(768, false),
227 KZC_ENTRY(1024, true),
228 KZC_ENTRY(1152, false),
229 KZC_ENTRY(1280, false),
230 KZC_ENTRY(1664, false),
231 KZC_ENTRY(2048, false),
232 KZC_ENTRY(4096, false),
233 KZC_ENTRY(6144, false),
234 KZC_ENTRY(8192, false),
235 KZC_ENTRY(16384, false),
236 KZC_ENTRY(32768, false),
237
238 #elif KALLOC_MINSIZE == 8 && KALLOC_LOG2_MINALIGN == 3
239 /* Zone config for embedded 32-bit platforms */
240 KZC_ENTRY(8, true),
241 KZC_ENTRY(16, true),
242 KZC_ENTRY(24, true),
243 KZC_ENTRY(32, true),
244 KZC_ENTRY(40, true),
245 KZC_ENTRY(48, true),
246 KZC_ENTRY(64, true),
247 KZC_ENTRY(72, true),
248 KZC_ENTRY(88, true),
249 KZC_ENTRY(112, true),
250 KZC_ENTRY(128, true),
251 KZC_ENTRY(192, true),
252 KZC_ENTRY(256, true),
253 KZC_ENTRY(288, true),
254 KZC_ENTRY(384, true),
255 KZC_ENTRY(440, true),
256 KZC_ENTRY(512, true),
257 KZC_ENTRY(576, false),
258 KZC_ENTRY(768, false),
259 KZC_ENTRY(1024, true),
260 KZC_ENTRY(1152, false),
261 KZC_ENTRY(1280, false),
262 KZC_ENTRY(1536, false),
263 KZC_ENTRY(2048, false),
264 KZC_ENTRY(2128, false),
265 KZC_ENTRY(3072, false),
266 KZC_ENTRY(4096, false),
267 KZC_ENTRY(6144, false),
268 KZC_ENTRY(8192, false),
269 /* To limit internal fragmentation, only add the following zones if the
270 * page size is greater than 4K.
271 * Note that we use ARM_PGBYTES here (instead of one of the VM macros)
272 * since it's guaranteed to be a compile time constant.
273 */
274 #if ARM_PGBYTES > 4096
275 KZC_ENTRY(16384, false),
276 KZC_ENTRY(32768, false),
277 #endif /* ARM_PGBYTES > 4096 */
278
279 #else
280 #error missing or invalid zone size parameters for kalloc
281 #endif
282
283 #else /* !defined(XNU_TARGET_OS_OSX) */
284
285 /* Zone config for macOS 64-bit platforms */
286 KZC_ENTRY(16, true),
287 KZC_ENTRY(32, true),
288 KZC_ENTRY(48, true),
289 KZC_ENTRY(64, true),
290 KZC_ENTRY(80, true),
291 KZC_ENTRY(96, true),
292 KZC_ENTRY(128, true),
293 KZC_ENTRY(160, true),
294 KZC_ENTRY(192, true),
295 KZC_ENTRY(224, true),
296 KZC_ENTRY(256, true),
297 KZC_ENTRY(288, true),
298 KZC_ENTRY(368, true),
299 KZC_ENTRY(400, true),
300 KZC_ENTRY(512, true),
301 KZC_ENTRY(576, true),
302 KZC_ENTRY(768, true),
303 KZC_ENTRY(1024, true),
304 KZC_ENTRY(1152, false),
305 KZC_ENTRY(1280, false),
306 KZC_ENTRY(1664, false),
307 KZC_ENTRY(2048, true),
308 KZC_ENTRY(4096, true),
309 KZC_ENTRY(6144, false),
310 KZC_ENTRY(8192, true),
311 KZC_ENTRY(12288, false),
312 KZC_ENTRY(16384, false)
313
314 #endif /* !defined(XNU_TARGET_OS_OSX) */
315
316 #undef KZC_ENTRY
317 };
318
319 #define MAX_K_ZONE(kzc) (uint32_t)(sizeof(kzc) / sizeof(kzc[0]))
320
321 /*
322 * Many kalloc() allocations are for small structures containing a few
323 * pointers and longs - the dlut[] direct lookup table, indexed by
324 * size normalized to the minimum alignment, finds the right zone index
325 * for them in one dereference.
326 */
327
328 #define INDEX_ZDLUT(size) (((size) + KALLOC_MINALIGN - 1) / KALLOC_MINALIGN)
329 #define MAX_SIZE_ZDLUT ((KALLOC_DLUT_SIZE - 1) * KALLOC_MINALIGN)
330
331 static SECURITY_READ_ONLY_LATE(zone_t) k_zone_default[MAX_K_ZONE(k_zone_cfg)];
332 static SECURITY_READ_ONLY_LATE(zone_t) k_zone_data_buffers[MAX_K_ZONE(k_zone_cfg)];
333 static SECURITY_READ_ONLY_LATE(zone_t) k_zone_kext[MAX_K_ZONE(k_zone_cfg)];
334
335 #if VM_MAX_TAG_ZONES
336 #if __LP64__
337 static_assert(VM_MAX_TAG_ZONES >=
338 MAX_K_ZONE(k_zone_cfg) + MAX_K_ZONE(k_zone_cfg) + MAX_K_ZONE(k_zone_cfg));
339 #else
340 static_assert(VM_MAX_TAG_ZONES >= MAX_K_ZONE(k_zone_cfg));
341 #endif
342 #endif
343
344 const char * const kalloc_heap_names[] = {
345 [KHEAP_ID_NONE] = "",
346 [KHEAP_ID_DEFAULT] = "default.",
347 [KHEAP_ID_DATA_BUFFERS] = "data.",
348 [KHEAP_ID_KEXT] = "kext.",
349 };
350
351 /*
352 * Default kalloc heap configuration
353 */
354 static SECURITY_READ_ONLY_LATE(struct kheap_zones) kalloc_zones_default = {
355 .cfg = k_zone_cfg,
356 .heap_id = KHEAP_ID_DEFAULT,
357 .k_zone = k_zone_default,
358 .max_k_zone = MAX_K_ZONE(k_zone_cfg)
359 };
360 SECURITY_READ_ONLY_LATE(struct kalloc_heap) KHEAP_DEFAULT[1] = {
361 {
362 .kh_zones = &kalloc_zones_default,
363 .kh_name = "default.",
364 .kh_heap_id = KHEAP_ID_DEFAULT,
365 }
366 };
367
368 KALLOC_HEAP_DEFINE(KHEAP_TEMP, "temp allocations", KHEAP_ID_DEFAULT);
369
370
371 /*
372 * Bag of bytes heap configuration
373 */
374 static SECURITY_READ_ONLY_LATE(struct kheap_zones) kalloc_zones_data_buffers = {
375 .cfg = k_zone_cfg,
376 .heap_id = KHEAP_ID_DATA_BUFFERS,
377 .k_zone = k_zone_data_buffers,
378 .max_k_zone = MAX_K_ZONE(k_zone_cfg)
379 };
380 SECURITY_READ_ONLY_LATE(struct kalloc_heap) KHEAP_DATA_BUFFERS[1] = {
381 {
382 .kh_zones = &kalloc_zones_data_buffers,
383 .kh_name = "data.",
384 .kh_heap_id = KHEAP_ID_DATA_BUFFERS,
385 }
386 };
387
388
389 /*
390 * Kext heap configuration
391 */
392 static SECURITY_READ_ONLY_LATE(struct kheap_zones) kalloc_zones_kext = {
393 .cfg = k_zone_cfg,
394 .heap_id = KHEAP_ID_KEXT,
395 .k_zone = k_zone_kext,
396 .max_k_zone = MAX_K_ZONE(k_zone_cfg)
397 };
398 SECURITY_READ_ONLY_LATE(struct kalloc_heap) KHEAP_KEXT[1] = {
399 {
400 .kh_zones = &kalloc_zones_kext,
401 .kh_name = "kext.",
402 .kh_heap_id = KHEAP_ID_KEXT,
403 }
404 };
405
406 KALLOC_HEAP_DEFINE(KERN_OS_MALLOC, "kern_os_malloc", KHEAP_ID_KEXT);
407
408 /*
409 * Initialize kalloc heap: Create zones, generate direct lookup table and
410 * do a quick test on lookups
411 */
412 __startup_func
413 static void
414 kalloc_zones_init(struct kheap_zones *zones)
415 {
416 struct kalloc_zone_cfg *cfg = zones->cfg;
417 zone_t *k_zone = zones->k_zone;
418 vm_size_t size;
419
420 /*
421 * Allocate a zone for each size we are going to handle.
422 */
423 for (uint32_t i = 0; i < zones->max_k_zone &&
424 (size = cfg[i].kzc_size) < kalloc_max; i++) {
425 zone_create_flags_t flags = ZC_KASAN_NOREDZONE |
426 ZC_KASAN_NOQUARANTINE | ZC_KALLOC_HEAP;
427 if (cfg[i].kzc_caching) {
428 flags |= ZC_CACHING;
429 }
430
431 k_zone[i] = zone_create_ext(cfg[i].kzc_name, size, flags,
432 ZONE_ID_ANY, ^(zone_t z){
433 z->kalloc_heap = zones->heap_id;
434 });
435 /*
436 * Set the updated elem size back to the config
437 */
438 cfg[i].kzc_size = k_zone[i]->z_elem_size;
439 }
440
441 /*
442 * Count all the "raw" views for zones in the heap.
443 */
444 zone_view_count += zones->max_k_zone;
445
446 /*
447 * Build the Direct LookUp Table for small allocations
448 * As k_zone_cfg is shared between the heaps the
449 * Direct LookUp Table is also shared and doesn't need to
450 * be rebuilt per heap.
451 */
452 size = 0;
453 for (int i = 0; i <= KALLOC_DLUT_SIZE; i++, size += KALLOC_MINALIGN) {
454 uint8_t zindex = 0;
455
456 while ((vm_size_t)(cfg[zindex].kzc_size) < size) {
457 zindex++;
458 }
459
460 if (i == KALLOC_DLUT_SIZE) {
461 zones->k_zindex_start = zindex;
462 break;
463 }
464 zones->dlut[i] = zindex;
465 }
466
467 #ifdef KALLOC_DEBUG
468 printf("kalloc_init: k_zindex_start %d\n", zones->k_zindex_start);
469
470 /*
471 * Do a quick synthesis to see how well/badly we can
472 * find-a-zone for a given size.
473 * Useful when debugging/tweaking the array of zone sizes.
474 * Cache misses probably more critical than compare-branches!
475 */
476 for (uint32_t i = 0; i < zones->max_k_zone; i++) {
477 vm_size_t testsize = (vm_size_t)(cfg[i].kzc_size - 1);
478 int compare = 0;
479 uint8_t zindex;
480
481 if (testsize < MAX_SIZE_ZDLUT) {
482 compare += 1; /* 'if' (T) */
483
484 long dindex = INDEX_ZDLUT(testsize);
485 zindex = (int)zones->dlut[dindex];
486 } else if (testsize < kalloc_max_prerounded) {
487 compare += 2; /* 'if' (F), 'if' (T) */
488
489 zindex = zones->k_zindex_start;
490 while ((vm_size_t)(cfg[zindex].kzc_size) < testsize) {
491 zindex++;
492 compare++; /* 'while' (T) */
493 }
494 compare++; /* 'while' (F) */
495 } else {
496 break; /* not zone-backed */
497 }
498 zone_t z = k_zone[zindex];
499 printf("kalloc_init: req size %4lu: %8s.%16s took %d compare%s\n",
500 (unsigned long)testsize, kalloc_heap_names[zones->heap_id],
501 z->z_name, compare, compare == 1 ? "" : "s");
502 }
503 #endif
504 }
505
506 /*
507 * Initialize the memory allocator. This should be called only
508 * once on a system wide basis (i.e. first processor to get here
509 * does the initialization).
510 *
511 * This initializes all of the zones.
512 */
513
514 __startup_func
515 static void
516 kalloc_init(void)
517 {
518 kern_return_t retval;
519 vm_offset_t min;
520 vm_size_t kalloc_map_size;
521 vm_map_kernel_flags_t vmk_flags;
522
523 /*
524 * Scale the kalloc_map_size to physical memory size: stay below
525 * 1/8th the total zone map size, or 128 MB (for a 32-bit kernel).
526 */
527 kalloc_map_size = (vm_size_t)(sane_size >> 5);
528 #if !__LP64__
529 if (kalloc_map_size > KALLOC_MAP_SIZE_MAX) {
530 kalloc_map_size = KALLOC_MAP_SIZE_MAX;
531 }
532 #endif /* !__LP64__ */
533 if (kalloc_map_size < KALLOC_MAP_SIZE_MIN) {
534 kalloc_map_size = KALLOC_MAP_SIZE_MIN;
535 }
536
537 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
538 vmk_flags.vmkf_permanent = TRUE;
539
540 retval = kmem_suballoc(kernel_map, &min, kalloc_map_size,
541 FALSE, VM_FLAGS_ANYWHERE, vmk_flags,
542 VM_KERN_MEMORY_KALLOC, &kalloc_map);
543
544 if (retval != KERN_SUCCESS) {
545 panic("kalloc_init: kmem_suballoc failed");
546 }
547
548 kalloc_map_min = min;
549 kalloc_map_max = min + kalloc_map_size - 1;
550
551 struct kheap_zones *khz_default = &kalloc_zones_default;
552 kalloc_max = (khz_default->cfg[khz_default->max_k_zone - 1].kzc_size << 1);
553 if (kalloc_max < KiB(16)) {
554 kalloc_max = KiB(16);
555 }
556 assert(kalloc_max <= KiB(64)); /* assumption made in size arrays */
557
558 kalloc_max_prerounded = kalloc_max / 2 + 1;
559 /* allocations larger than 16 times kalloc_max go directly to kernel map */
560 kalloc_kernmap_size = (kalloc_max * 16) + 1;
561 kalloc_largest_allocated = kalloc_kernmap_size;
562
563 /* Initialize kalloc default heap */
564 kalloc_zones_init(&kalloc_zones_default);
565
566 /* Initialize kalloc data buffers heap */
567 if (ZSECURITY_OPTIONS_SUBMAP_USER_DATA & zsecurity_options) {
568 kalloc_zones_init(&kalloc_zones_data_buffers);
569 } else {
570 *KHEAP_DATA_BUFFERS = *KHEAP_DEFAULT;
571 }
572
573 /* Initialize kalloc kext heap */
574 if (ZSECURITY_OPTIONS_SEQUESTER_KEXT_KALLOC & zsecurity_options) {
575 kalloc_zones_init(&kalloc_zones_kext);
576 } else {
577 *KHEAP_KEXT = *KHEAP_DEFAULT;
578 }
579 }
580 STARTUP(ZALLOC, STARTUP_RANK_THIRD, kalloc_init);
581
582
583 #pragma mark accessors
584
585 static void
586 KALLOC_ZINFO_SALLOC(vm_size_t bytes)
587 {
588 thread_t thr = current_thread();
589 ledger_debit_thread(thr, thr->t_ledger, task_ledgers.tkm_shared, bytes);
590 }
591
592 static void
593 KALLOC_ZINFO_SFREE(vm_size_t bytes)
594 {
595 thread_t thr = current_thread();
596 ledger_credit_thread(thr, thr->t_ledger, task_ledgers.tkm_shared, bytes);
597 }
598
599 static inline vm_map_t
600 kalloc_map_for_addr(vm_address_t addr)
601 {
602 if (addr >= kalloc_map_min && addr < kalloc_map_max) {
603 return kalloc_map;
604 }
605 return kernel_map;
606 }
607
608 static inline vm_map_t
609 kalloc_map_for_size(vm_size_t size)
610 {
611 if (size < kalloc_kernmap_size) {
612 return kalloc_map;
613 }
614 return kernel_map;
615 }
616
617 zone_t
618 kalloc_heap_zone_for_size(kalloc_heap_t kheap, vm_size_t size)
619 {
620 struct kheap_zones *khz = kheap->kh_zones;
621
622 if (size < MAX_SIZE_ZDLUT) {
623 uint32_t zindex = khz->dlut[INDEX_ZDLUT(size)];
624 return khz->k_zone[zindex];
625 }
626
627 if (size < kalloc_max_prerounded) {
628 uint32_t zindex = khz->k_zindex_start;
629 while (khz->cfg[zindex].kzc_size < size) {
630 zindex++;
631 }
632 assert(zindex < khz->max_k_zone);
633 return khz->k_zone[zindex];
634 }
635
636 return ZONE_NULL;
637 }
638
639 static vm_size_t
640 vm_map_lookup_kalloc_entry_locked(vm_map_t map, void *addr)
641 {
642 vm_map_entry_t vm_entry = NULL;
643
644 if (!vm_map_lookup_entry(map, (vm_map_offset_t)addr, &vm_entry)) {
645 panic("address %p not allocated via kalloc, map %p",
646 addr, map);
647 }
648 if (vm_entry->vme_start != (vm_map_offset_t)addr) {
649 panic("address %p inside vm entry %p [%p:%p), map %p",
650 addr, vm_entry, (void *)vm_entry->vme_start,
651 (void *)vm_entry->vme_end, map);
652 }
653 if (!vm_entry->vme_atomic) {
654 panic("address %p not managed by kalloc (entry %p, map %p)",
655 addr, vm_entry, map);
656 }
657 return vm_entry->vme_end - vm_entry->vme_start;
658 }
659
660 #if KASAN_KALLOC
661 /*
662 * KASAN kalloc stashes the original user-requested size away in the poisoned
663 * area. Return that directly.
664 */
665 vm_size_t
666 kalloc_size(void *addr)
667 {
668 (void)vm_map_lookup_kalloc_entry_locked; /* silence warning */
669 return kasan_user_size((vm_offset_t)addr);
670 }
671 #else
672 vm_size_t
673 kalloc_size(void *addr)
674 {
675 vm_map_t map;
676 vm_size_t size;
677
678 size = zone_element_size(addr, NULL);
679 if (size) {
680 return size;
681 }
682
683 map = kalloc_map_for_addr((vm_offset_t)addr);
684 vm_map_lock_read(map);
685 size = vm_map_lookup_kalloc_entry_locked(map, addr);
686 vm_map_unlock_read(map);
687 return size;
688 }
689 #endif
690
691 vm_size_t
692 kalloc_bucket_size(vm_size_t size)
693 {
694 zone_t z = kalloc_heap_zone_for_size(KHEAP_DEFAULT, size);
695 vm_map_t map = kalloc_map_for_size(size);
696
697 if (z) {
698 return zone_elem_size(z);
699 }
700 return vm_map_round_page(size, VM_MAP_PAGE_MASK(map));
701 }
702
703 #pragma mark kalloc
704
705 void
706 kheap_temp_leak_panic(thread_t self)
707 {
708 #if DEBUG || DEVELOPMENT
709 if (__improbable(kheap_temp_debug)) {
710 struct kheap_temp_header *hdr = qe_dequeue_head(&self->t_temp_alloc_list,
711 struct kheap_temp_header, kht_hdr_link);
712
713 panic_plain("KHEAP_TEMP leak on thread %p (%d), allocated at:\n"
714 " %#016lx\n" " %#016lx\n" " %#016lx\n" " %#016lx\n"
715 " %#016lx\n" " %#016lx\n" " %#016lx\n" " %#016lx\n"
716 " %#016lx\n" " %#016lx\n" " %#016lx\n" " %#016lx\n"
717 " %#016lx\n" " %#016lx\n",
718 self, self->t_temp_alloc_count,
719 hdr->kht_hdr_pcs[0], hdr->kht_hdr_pcs[1],
720 hdr->kht_hdr_pcs[2], hdr->kht_hdr_pcs[3],
721 hdr->kht_hdr_pcs[4], hdr->kht_hdr_pcs[5],
722 hdr->kht_hdr_pcs[6], hdr->kht_hdr_pcs[7],
723 hdr->kht_hdr_pcs[8], hdr->kht_hdr_pcs[9],
724 hdr->kht_hdr_pcs[10], hdr->kht_hdr_pcs[11],
725 hdr->kht_hdr_pcs[12], hdr->kht_hdr_pcs[13]);
726 }
727 panic("KHEAP_TEMP leak on thread %p (%d) "
728 "(boot with kheap_temp_debug=1 to debug)",
729 self, self->t_temp_alloc_count);
730 #else /* !DEBUG && !DEVELOPMENT */
731 panic("KHEAP_TEMP leak on thread %p (%d)",
732 self, self->t_temp_alloc_count);
733 #endif /* !DEBUG && !DEVELOPMENT */
734 }
735
736 __abortlike
737 static void
738 kheap_temp_overuse_panic(thread_t self)
739 {
740 panic("too many KHEAP_TEMP allocations in flight: %d",
741 self->t_temp_alloc_count);
742 }
743
744 __attribute__((noinline))
745 static struct kalloc_result
746 kalloc_large(
747 kalloc_heap_t kheap,
748 vm_size_t req_size,
749 vm_size_t size,
750 zalloc_flags_t flags,
751 vm_allocation_site_t *site)
752 {
753 int kma_flags = KMA_ATOMIC | KMA_KOBJECT;
754 vm_tag_t tag = VM_KERN_MEMORY_KALLOC;
755 vm_map_t alloc_map;
756 vm_offset_t addr;
757
758 if (flags & Z_NOFAIL) {
759 panic("trying to kalloc(Z_NOFAIL) with a large size (%zd)",
760 (size_t)size);
761 }
762 /* kmem_alloc could block so we return if noblock */
763 if (flags & Z_NOWAIT) {
764 return (struct kalloc_result){ };
765 }
766
767 if (flags & Z_NOPAGEWAIT) {
768 kma_flags |= KMA_NOPAGEWAIT;
769 }
770 if (flags & Z_ZERO) {
771 kma_flags |= KMA_ZERO;
772 }
773
774 #if KASAN_KALLOC
775 /* large allocation - use guard pages instead of small redzones */
776 size = round_page(req_size + 2 * PAGE_SIZE);
777 assert(size >= MAX_SIZE_ZDLUT && size >= kalloc_max_prerounded);
778 #else
779 size = round_page(size);
780 #endif
781
782 alloc_map = kalloc_map_for_size(size);
783
784 if (site) {
785 tag = vm_tag_alloc(site);
786 }
787
788 if (kmem_alloc_flags(alloc_map, &addr, size, tag, kma_flags) != KERN_SUCCESS) {
789 if (alloc_map != kernel_map) {
790 if (kalloc_fallback_count++ == 0) {
791 printf("%s: falling back to kernel_map\n", __func__);
792 }
793 if (kmem_alloc_flags(kernel_map, &addr, size, tag, kma_flags) != KERN_SUCCESS) {
794 addr = 0;
795 }
796 } else {
797 addr = 0;
798 }
799 }
800
801 if (addr != 0) {
802 kalloc_spin_lock();
803 /*
804 * Thread-safe version of the workaround for 4740071
805 * (a double FREE())
806 */
807 if (size > kalloc_largest_allocated) {
808 kalloc_largest_allocated = size;
809 }
810
811 kalloc_large_inuse++;
812 assert(kalloc_large_total + size >= kalloc_large_total); /* no wrap around */
813 kalloc_large_total += size;
814 kalloc_large_sum += size;
815
816 if (kalloc_large_total > kalloc_large_max) {
817 kalloc_large_max = kalloc_large_total;
818 }
819
820 kalloc_unlock();
821
822 KALLOC_ZINFO_SALLOC(size);
823 }
824 #if KASAN_KALLOC
825 /* fixup the return address to skip the redzone */
826 addr = kasan_alloc(addr, size, req_size, PAGE_SIZE);
827 /*
828 * Initialize buffer with unique pattern only if memory
829 * wasn't expected to be zeroed.
830 */
831 if (!(flags & Z_ZERO)) {
832 kasan_leak_init(addr, req_size);
833 }
834 #else
835 req_size = size;
836 #endif
837
838 if (addr && kheap == KHEAP_TEMP) {
839 thread_t self = current_thread();
840
841 if (self->t_temp_alloc_count++ > UINT16_MAX) {
842 kheap_temp_overuse_panic(self);
843 }
844 #if DEBUG || DEVELOPMENT
845 if (__improbable(kheap_temp_debug)) {
846 struct kheap_temp_header *hdr = (void *)addr;
847 enqueue_head(&self->t_temp_alloc_list,
848 &hdr->kht_hdr_link);
849 backtrace(hdr->kht_hdr_pcs, KHT_BT_COUNT, NULL);
850 req_size -= sizeof(struct kheap_temp_header);
851 addr += sizeof(struct kheap_temp_header);
852 }
853 #endif /* DEBUG || DEVELOPMENT */
854 }
855
856 DTRACE_VM3(kalloc, vm_size_t, size, vm_size_t, req_size, void*, addr);
857 return (struct kalloc_result){ .addr = (void *)addr, .size = req_size };
858 }
859
860 struct kalloc_result
861 kalloc_ext(
862 kalloc_heap_t kheap,
863 vm_size_t req_size,
864 zalloc_flags_t flags,
865 vm_allocation_site_t *site)
866 {
867 vm_tag_t tag = VM_KERN_MEMORY_KALLOC;
868 vm_size_t size;
869 void *addr;
870 zone_t z;
871
872 #if DEBUG || DEVELOPMENT
873 if (__improbable(kheap_temp_debug)) {
874 if (kheap == KHEAP_TEMP) {
875 req_size += sizeof(struct kheap_temp_header);
876 }
877 }
878 #endif /* DEBUG || DEVELOPMENT */
879
880 /*
881 * Kasan for kalloc heaps will put the redzones *inside*
882 * the allocation, and hence augment its size.
883 *
884 * kalloc heaps do not use zone_t::kasan_redzone.
885 */
886 #if KASAN_KALLOC
887 size = kasan_alloc_resize(req_size);
888 #else
889 size = req_size;
890 #endif
891 z = kalloc_heap_zone_for_size(kheap, size);
892 if (__improbable(z == ZONE_NULL)) {
893 return kalloc_large(kheap, req_size, size, flags, site);
894 }
895
896 #ifdef KALLOC_DEBUG
897 if (size > zone_elem_size(z)) {
898 panic("%s: z %p (%s%s) but requested size %lu", __func__, z,
899 kalloc_heap_names[kheap->kh_zones->heap_id], z->z_name,
900 (unsigned long)size);
901 }
902 #endif
903 assert(size <= zone_elem_size(z));
904
905 #if VM_MAX_TAG_ZONES
906 if (z->tags && site) {
907 tag = vm_tag_alloc(site);
908 if ((flags & (Z_NOWAIT | Z_NOPAGEWAIT)) && !vm_allocation_zone_totals[tag]) {
909 tag = VM_KERN_MEMORY_KALLOC;
910 }
911 }
912 #endif
913 addr = zalloc_ext(z, kheap->kh_stats ?: z->z_stats,
914 flags | Z_VM_TAG(tag), zone_elem_size(z) - size);
915
916 #if KASAN_KALLOC
917 addr = (void *)kasan_alloc((vm_offset_t)addr, zone_elem_size(z),
918 req_size, KASAN_GUARD_SIZE);
919 #else
920 req_size = zone_elem_size(z);
921 #endif
922
923 if (addr && kheap == KHEAP_TEMP) {
924 thread_t self = current_thread();
925
926 if (self->t_temp_alloc_count++ > UINT16_MAX) {
927 kheap_temp_overuse_panic(self);
928 }
929 #if DEBUG || DEVELOPMENT
930 if (__improbable(kheap_temp_debug)) {
931 struct kheap_temp_header *hdr = (void *)addr;
932 enqueue_head(&self->t_temp_alloc_list,
933 &hdr->kht_hdr_link);
934 backtrace(hdr->kht_hdr_pcs, KHT_BT_COUNT, NULL);
935 req_size -= sizeof(struct kheap_temp_header);
936 addr += sizeof(struct kheap_temp_header);
937 }
938 #endif /* DEBUG || DEVELOPMENT */
939 }
940
941 DTRACE_VM3(kalloc, vm_size_t, size, vm_size_t, req_size, void*, addr);
942 return (struct kalloc_result){ .addr = addr, .size = req_size };
943 }
944
945 void *
946 kalloc_external(vm_size_t size);
947 void *
948 kalloc_external(vm_size_t size)
949 {
950 return kheap_alloc_tag_bt(KHEAP_KEXT, size, Z_WAITOK, VM_KERN_MEMORY_KALLOC);
951 }
952
953
954 #pragma mark kfree
955
956 __attribute__((noinline))
957 static void
958 kfree_large(vm_offset_t addr, vm_size_t size)
959 {
960 vm_map_t map = kalloc_map_for_addr(addr);
961 kern_return_t ret;
962 vm_offset_t end;
963
964 if (addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS ||
965 os_add_overflow(addr, size, &end) ||
966 end > VM_MAX_KERNEL_ADDRESS) {
967 panic("kfree: address range (%p, %ld) doesn't belong to the kernel",
968 (void *)addr, (uintptr_t)size);
969 }
970
971 if (size == 0) {
972 vm_map_lock(map);
973 size = vm_map_lookup_kalloc_entry_locked(map, (void *)addr);
974 ret = vm_map_remove_locked(map,
975 vm_map_trunc_page(addr, VM_MAP_PAGE_MASK(map)),
976 vm_map_round_page(addr + size, VM_MAP_PAGE_MASK(map)),
977 VM_MAP_REMOVE_KUNWIRE);
978 if (ret != KERN_SUCCESS) {
979 panic("kfree: vm_map_remove_locked() failed for "
980 "addr: %p, map: %p ret: %d", (void *)addr, map, ret);
981 }
982 vm_map_unlock(map);
983 } else {
984 size = round_page(size);
985
986 if (size > kalloc_largest_allocated) {
987 panic("kfree: size %lu > kalloc_largest_allocated %lu",
988 (uintptr_t)size, (uintptr_t)kalloc_largest_allocated);
989 }
990 kmem_free(map, addr, size);
991 }
992
993 kalloc_spin_lock();
994
995 assert(kalloc_large_total >= size);
996 kalloc_large_total -= size;
997 kalloc_large_inuse--;
998
999 kalloc_unlock();
1000
1001 #if !KASAN_KALLOC
1002 DTRACE_VM3(kfree, vm_size_t, size, vm_size_t, size, void*, addr);
1003 #endif
1004
1005 KALLOC_ZINFO_SFREE(size);
1006 return;
1007 }
1008
1009 __abortlike
1010 static void
1011 kfree_heap_confusion_panic(kalloc_heap_t kheap, void *data, size_t size, zone_t z)
1012 {
1013 if (z->kalloc_heap == KHEAP_ID_NONE) {
1014 panic("kfree: addr %p, size %zd found in regular zone '%s%s'",
1015 data, size, zone_heap_name(z), z->z_name);
1016 } else {
1017 panic("kfree: addr %p, size %zd found in heap %s* instead of %s*",
1018 data, size, zone_heap_name(z),
1019 kalloc_heap_names[kheap->kh_heap_id]);
1020 }
1021 }
1022
1023 __abortlike
1024 static void
1025 kfree_size_confusion_panic(zone_t z, void *data, size_t size, size_t zsize)
1026 {
1027 if (z) {
1028 panic("kfree: addr %p, size %zd found in zone '%s%s' "
1029 "with elem_size %zd",
1030 data, size, zone_heap_name(z), z->z_name, zsize);
1031 } else {
1032 panic("kfree: addr %p, size %zd not found in any zone",
1033 data, size);
1034 }
1035 }
1036
1037 __abortlike
1038 static void
1039 kfree_size_invalid_panic(void *data, size_t size)
1040 {
1041 panic("kfree: addr %p trying to free with nonsensical size %zd",
1042 data, size);
1043 }
1044
1045 __abortlike
1046 static void
1047 krealloc_size_invalid_panic(void *data, size_t size)
1048 {
1049 panic("krealloc: addr %p trying to free with nonsensical size %zd",
1050 data, size);
1051 }
1052
1053 __abortlike
1054 static void
1055 kfree_temp_imbalance_panic(void *data, size_t size)
1056 {
1057 panic("kfree: KHEAP_TEMP allocation imbalance freeing addr %p, size %zd",
1058 data, size);
1059 }
1060
1061 /* used to implement kheap_free_addr() */
1062 #define KFREE_UNKNOWN_SIZE ((vm_size_t)~0)
1063 #define KFREE_ABSURD_SIZE \
1064 ((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_AND_KEXT_ADDRESS) / 2)
1065
1066 static void
1067 kfree_ext(kalloc_heap_t kheap, void *data, vm_size_t size)
1068 {
1069 zone_stats_t zs = NULL;
1070 zone_t z;
1071 vm_size_t zsize;
1072
1073 if (__improbable(data == NULL)) {
1074 return;
1075 }
1076
1077 if (kheap == KHEAP_TEMP) {
1078 assert(size != KFREE_UNKNOWN_SIZE);
1079 if (current_thread()->t_temp_alloc_count-- == 0) {
1080 kfree_temp_imbalance_panic(data, size);
1081 }
1082 #if DEBUG || DEVELOPMENT
1083 if (__improbable(kheap_temp_debug)) {
1084 size += sizeof(struct kheap_temp_header);
1085 data -= sizeof(struct kheap_temp_header);
1086 remqueue(&((struct kheap_temp_header *)data)->kht_hdr_link);
1087 }
1088 #endif /* DEBUG || DEVELOPMENT */
1089 }
1090
1091 #if KASAN_KALLOC
1092 /*
1093 * Resize back to the real allocation size and hand off to the KASan
1094 * quarantine. `data` may then point to a different allocation.
1095 */
1096 vm_size_t user_size = size;
1097 if (size == KFREE_UNKNOWN_SIZE) {
1098 user_size = size = kalloc_size(data);
1099 }
1100 kasan_check_free((vm_address_t)data, size, KASAN_HEAP_KALLOC);
1101 data = (void *)kasan_dealloc((vm_address_t)data, &size);
1102 kasan_free(&data, &size, KASAN_HEAP_KALLOC, NULL, user_size, true);
1103 if (!data) {
1104 return;
1105 }
1106 #endif
1107
1108 if (size >= kalloc_max_prerounded && size != KFREE_UNKNOWN_SIZE) {
1109 return kfree_large((vm_offset_t)data, size);
1110 }
1111
1112 zsize = zone_element_size(data, &z);
1113 if (size == KFREE_UNKNOWN_SIZE) {
1114 if (zsize == 0) {
1115 return kfree_large((vm_offset_t)data, 0);
1116 }
1117 size = zsize;
1118 } else if (size > zsize) {
1119 kfree_size_confusion_panic(z, data, size, zsize);
1120 }
1121
1122 if (kheap != KHEAP_ANY) {
1123 if (kheap->kh_heap_id != z->kalloc_heap) {
1124 kfree_heap_confusion_panic(kheap, data, size, z);
1125 }
1126 zs = kheap->kh_stats;
1127 } else if (z->kalloc_heap != KHEAP_ID_DEFAULT &&
1128 z->kalloc_heap != KHEAP_ID_KEXT) {
1129 kfree_heap_confusion_panic(kheap, data, size, z);
1130 }
1131
1132 #if !KASAN_KALLOC
1133 DTRACE_VM3(kfree, vm_size_t, size, vm_size_t, zsize, void*, data);
1134 #endif
1135 zfree_ext(z, zs ?: z->z_stats, data);
1136 }
1137
1138 void
1139 (kfree)(void *addr, vm_size_t size)
1140 {
1141 if (size > KFREE_ABSURD_SIZE) {
1142 kfree_size_invalid_panic(addr, size);
1143 }
1144 kfree_ext(KHEAP_ANY, addr, size);
1145 }
1146
1147 void
1148 (kheap_free)(kalloc_heap_t kheap, void *addr, vm_size_t size)
1149 {
1150 if (size > KFREE_ABSURD_SIZE) {
1151 kfree_size_invalid_panic(addr, size);
1152 }
1153 kfree_ext(kheap, addr, size);
1154 }
1155
1156 void
1157 (kheap_free_addr)(kalloc_heap_t kheap, void *addr)
1158 {
1159 kfree_ext(kheap, addr, KFREE_UNKNOWN_SIZE);
1160 }
1161
1162 static struct kalloc_result
1163 _krealloc_ext(
1164 kalloc_heap_t kheap,
1165 void *addr,
1166 vm_size_t old_size,
1167 vm_size_t new_size,
1168 zalloc_flags_t flags,
1169 vm_allocation_site_t *site)
1170 {
1171 vm_size_t old_bucket_size, new_bucket_size, min_size;
1172 struct kalloc_result kr;
1173
1174 if (new_size == 0) {
1175 kfree_ext(kheap, addr, old_size);
1176 return (struct kalloc_result){ };
1177 }
1178
1179 if (addr == NULL) {
1180 return kalloc_ext(kheap, new_size, flags, site);
1181 }
1182
1183 /*
1184 * Find out the size of the bucket in which the new sized allocation
1185 * would land. If it matches the bucket of the original allocation,
1186 * simply return the same address.
1187 */
1188 new_bucket_size = kalloc_bucket_size(new_size);
1189 if (old_size == KFREE_UNKNOWN_SIZE) {
1190 old_size = old_bucket_size = kalloc_size(addr);
1191 } else {
1192 old_bucket_size = kalloc_bucket_size(old_size);
1193 }
1194 min_size = MIN(old_size, new_size);
1195
1196 if (old_bucket_size == new_bucket_size) {
1197 kr.addr = addr;
1198 #if KASAN_KALLOC
1199 kr.size = new_size;
1200 #else
1201 kr.size = new_bucket_size;
1202 #endif
1203 } else {
1204 kr = kalloc_ext(kheap, new_size, flags & ~Z_ZERO, site);
1205 if (kr.addr == NULL) {
1206 return kr;
1207 }
1208
1209 memcpy(kr.addr, addr, min_size);
1210 kfree_ext(kheap, addr, old_size);
1211 }
1212 if ((flags & Z_ZERO) && kr.size > min_size) {
1213 bzero(kr.addr + min_size, kr.size - min_size);
1214 }
1215 return kr;
1216 }
1217
1218 struct kalloc_result
1219 krealloc_ext(
1220 kalloc_heap_t kheap,
1221 void *addr,
1222 vm_size_t old_size,
1223 vm_size_t new_size,
1224 zalloc_flags_t flags,
1225 vm_allocation_site_t *site)
1226 {
1227 if (old_size > KFREE_ABSURD_SIZE) {
1228 krealloc_size_invalid_panic(addr, old_size);
1229 }
1230 return _krealloc_ext(kheap, addr, old_size, new_size, flags, site);
1231 }
1232
1233 struct kalloc_result
1234 kheap_realloc_addr(
1235 kalloc_heap_t kheap,
1236 void *addr,
1237 vm_size_t size,
1238 zalloc_flags_t flags,
1239 vm_allocation_site_t *site)
1240 {
1241 return _krealloc_ext(kheap, addr, KFREE_UNKNOWN_SIZE, size, flags, site);
1242 }
1243
1244 __startup_func
1245 void
1246 kheap_startup_init(kalloc_heap_t kheap)
1247 {
1248 struct kheap_zones *zones;
1249
1250 switch (kheap->kh_heap_id) {
1251 case KHEAP_ID_DEFAULT:
1252 zones = KHEAP_DEFAULT->kh_zones;
1253 break;
1254 case KHEAP_ID_DATA_BUFFERS:
1255 zones = KHEAP_DATA_BUFFERS->kh_zones;
1256 break;
1257 case KHEAP_ID_KEXT:
1258 zones = KHEAP_KEXT->kh_zones;
1259 break;
1260 default:
1261 panic("kalloc_heap_startup_init: invalid KHEAP_ID: %d",
1262 kheap->kh_heap_id);
1263 }
1264
1265 kheap->kh_heap_id = zones->heap_id;
1266 kheap->kh_zones = zones;
1267 kheap->kh_stats = zalloc_percpu_permanent_type(struct zone_stats);
1268 kheap->kh_next = zones->views;
1269 zones->views = kheap;
1270
1271 zone_view_count += 1;
1272 }
1273
1274 #pragma mark OSMalloc
1275 /*
1276 * This is a deprecated interface, here only for legacy reasons.
1277 * There is no internal variant of any of these symbols on purpose.
1278 */
1279 #define OSMallocDeprecated
1280 #include <libkern/OSMalloc.h>
1281
1282 static KALLOC_HEAP_DEFINE(OSMALLOC, "osmalloc", KHEAP_ID_KEXT);
1283 static queue_head_t OSMalloc_tag_list = QUEUE_HEAD_INITIALIZER(OSMalloc_tag_list);
1284 static LCK_GRP_DECLARE(OSMalloc_tag_lck_grp, "OSMalloc_tag");
1285 static LCK_SPIN_DECLARE(OSMalloc_tag_lock, &OSMalloc_tag_lck_grp);
1286
1287 #define OSMalloc_tag_spin_lock() lck_spin_lock(&OSMalloc_tag_lock)
1288 #define OSMalloc_tag_unlock() lck_spin_unlock(&OSMalloc_tag_lock)
1289
1290 extern typeof(OSMalloc_Tagalloc) OSMalloc_Tagalloc_external;
1291 OSMallocTag
1292 OSMalloc_Tagalloc_external(const char *str, uint32_t flags)
1293 {
1294 OSMallocTag OSMTag;
1295
1296 OSMTag = kheap_alloc(OSMALLOC, sizeof(*OSMTag), Z_WAITOK | Z_ZERO);
1297
1298 if (flags & OSMT_PAGEABLE) {
1299 OSMTag->OSMT_attr = OSMT_ATTR_PAGEABLE;
1300 }
1301
1302 OSMTag->OSMT_refcnt = 1;
1303
1304 strlcpy(OSMTag->OSMT_name, str, OSMT_MAX_NAME);
1305
1306 OSMalloc_tag_spin_lock();
1307 enqueue_tail(&OSMalloc_tag_list, (queue_entry_t)OSMTag);
1308 OSMalloc_tag_unlock();
1309 OSMTag->OSMT_state = OSMT_VALID;
1310 return OSMTag;
1311 }
1312
1313 static void
1314 OSMalloc_Tagref(OSMallocTag tag)
1315 {
1316 if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID)) {
1317 panic("OSMalloc_Tagref():'%s' has bad state 0x%08X\n",
1318 tag->OSMT_name, tag->OSMT_state);
1319 }
1320
1321 os_atomic_inc(&tag->OSMT_refcnt, relaxed);
1322 }
1323
1324 static void
1325 OSMalloc_Tagrele(OSMallocTag tag)
1326 {
1327 if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID)) {
1328 panic("OSMalloc_Tagref():'%s' has bad state 0x%08X\n",
1329 tag->OSMT_name, tag->OSMT_state);
1330 }
1331
1332 if (os_atomic_dec(&tag->OSMT_refcnt, relaxed) != 0) {
1333 return;
1334 }
1335
1336 if (os_atomic_cmpxchg(&tag->OSMT_state,
1337 OSMT_VALID | OSMT_RELEASED, OSMT_VALID | OSMT_RELEASED, acq_rel)) {
1338 OSMalloc_tag_spin_lock();
1339 (void)remque((queue_entry_t)tag);
1340 OSMalloc_tag_unlock();
1341 kheap_free(OSMALLOC, tag, sizeof(*tag));
1342 } else {
1343 panic("OSMalloc_Tagrele():'%s' has refcnt 0\n", tag->OSMT_name);
1344 }
1345 }
1346
1347 extern typeof(OSMalloc_Tagfree) OSMalloc_Tagfree_external;
1348 void
1349 OSMalloc_Tagfree_external(OSMallocTag tag)
1350 {
1351 if (!os_atomic_cmpxchg(&tag->OSMT_state,
1352 OSMT_VALID, OSMT_VALID | OSMT_RELEASED, acq_rel)) {
1353 panic("OSMalloc_Tagfree():'%s' has bad state 0x%08X \n",
1354 tag->OSMT_name, tag->OSMT_state);
1355 }
1356
1357 if (os_atomic_dec(&tag->OSMT_refcnt, relaxed) == 0) {
1358 OSMalloc_tag_spin_lock();
1359 (void)remque((queue_entry_t)tag);
1360 OSMalloc_tag_unlock();
1361 kheap_free(OSMALLOC, tag, sizeof(*tag));
1362 }
1363 }
1364
1365 extern typeof(OSMalloc) OSMalloc_external;
1366 void *
1367 OSMalloc_external(
1368 uint32_t size, OSMallocTag tag)
1369 {
1370 void *addr = NULL;
1371 kern_return_t kr;
1372
1373 OSMalloc_Tagref(tag);
1374 if ((tag->OSMT_attr & OSMT_PAGEABLE) && (size & ~PAGE_MASK)) {
1375 if ((kr = kmem_alloc_pageable_external(kernel_map,
1376 (vm_offset_t *)&addr, size)) != KERN_SUCCESS) {
1377 addr = NULL;
1378 }
1379 } else {
1380 addr = kheap_alloc_tag_bt(OSMALLOC, size,
1381 Z_WAITOK, VM_KERN_MEMORY_KALLOC);
1382 }
1383
1384 if (!addr) {
1385 OSMalloc_Tagrele(tag);
1386 }
1387
1388 return addr;
1389 }
1390
1391 extern typeof(OSMalloc_nowait) OSMalloc_nowait_external;
1392 void *
1393 OSMalloc_nowait_external(uint32_t size, OSMallocTag tag)
1394 {
1395 void *addr = NULL;
1396
1397 if (tag->OSMT_attr & OSMT_PAGEABLE) {
1398 return NULL;
1399 }
1400
1401 OSMalloc_Tagref(tag);
1402 /* XXX: use non-blocking kalloc for now */
1403 addr = kheap_alloc_tag_bt(OSMALLOC, (vm_size_t)size,
1404 Z_NOWAIT, VM_KERN_MEMORY_KALLOC);
1405 if (addr == NULL) {
1406 OSMalloc_Tagrele(tag);
1407 }
1408
1409 return addr;
1410 }
1411
1412 extern typeof(OSMalloc_noblock) OSMalloc_noblock_external;
1413 void *
1414 OSMalloc_noblock_external(uint32_t size, OSMallocTag tag)
1415 {
1416 void *addr = NULL;
1417
1418 if (tag->OSMT_attr & OSMT_PAGEABLE) {
1419 return NULL;
1420 }
1421
1422 OSMalloc_Tagref(tag);
1423 addr = kheap_alloc_tag_bt(OSMALLOC, (vm_size_t)size,
1424 Z_NOWAIT, VM_KERN_MEMORY_KALLOC);
1425 if (addr == NULL) {
1426 OSMalloc_Tagrele(tag);
1427 }
1428
1429 return addr;
1430 }
1431
1432 extern typeof(OSFree) OSFree_external;
1433 void
1434 OSFree_external(void *addr, uint32_t size, OSMallocTag tag)
1435 {
1436 if ((tag->OSMT_attr & OSMT_PAGEABLE)
1437 && (size & ~PAGE_MASK)) {
1438 kmem_free(kernel_map, (vm_offset_t)addr, size);
1439 } else {
1440 kheap_free(OSMALLOC, addr, size);
1441 }
1442
1443 OSMalloc_Tagrele(tag);
1444 }
1445
1446 #pragma mark kern_os_malloc
1447
1448 void *
1449 kern_os_malloc_external(size_t size);
1450 void *
1451 kern_os_malloc_external(size_t size)
1452 {
1453 if (size == 0) {
1454 return NULL;
1455 }
1456
1457 return kheap_alloc_tag_bt(KERN_OS_MALLOC, size, Z_WAITOK | Z_ZERO,
1458 VM_KERN_MEMORY_LIBKERN);
1459 }
1460
1461 void
1462 kern_os_free_external(void *addr);
1463 void
1464 kern_os_free_external(void *addr)
1465 {
1466 kheap_free_addr(KERN_OS_MALLOC, addr);
1467 }
1468
1469 void *
1470 kern_os_realloc_external(void *addr, size_t nsize);
1471 void *
1472 kern_os_realloc_external(void *addr, size_t nsize)
1473 {
1474 VM_ALLOC_SITE_STATIC(VM_TAG_BT, VM_KERN_MEMORY_LIBKERN);
1475
1476 return kheap_realloc_addr(KERN_OS_MALLOC, addr, nsize,
1477 Z_WAITOK | Z_ZERO, &site).addr;
1478 }
1479
1480 void
1481 kern_os_zfree(zone_t zone, void *addr, vm_size_t size)
1482 {
1483 if (zsecurity_options & ZSECURITY_OPTIONS_STRICT_IOKIT_FREE
1484 || zone_owns(zone, addr)) {
1485 zfree(zone, addr);
1486 } else {
1487 /*
1488 * Third party kexts might not know about the operator new
1489 * and be allocated from the KEXT heap
1490 */
1491 printf("kern_os_zfree: kheap_free called for object from zone %s\n",
1492 zone->z_name);
1493 kheap_free(KHEAP_KEXT, addr, size);
1494 }
1495 }
1496
1497 void
1498 kern_os_kfree(void *addr, vm_size_t size)
1499 {
1500 if (zsecurity_options & ZSECURITY_OPTIONS_STRICT_IOKIT_FREE) {
1501 kheap_free(KHEAP_DEFAULT, addr, size);
1502 } else {
1503 /*
1504 * Third party kexts may not know about newly added operator
1505 * default new/delete. If they call new for any iokit object
1506 * it will end up coming from the KEXT heap. If these objects
1507 * are freed by calling release() or free(), the internal
1508 * version of operator delete is called and the kernel ends
1509 * up freeing the object to the DEFAULT heap.
1510 */
1511 kheap_free(KHEAP_ANY, addr, size);
1512 }
1513 }
mirror of XNU