2 * Copyright (c) 2009 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #include <kern/backtrace.h>
30 #include <vm/vm_map_store_rb.h>
32 RB_GENERATE(rb_head
, vm_map_store
, entry
, rb_node_compare
);
34 #define VME_FOR_STORE( store) \
35 (vm_map_entry_t)(((unsigned long)store) - ((unsigned long)sizeof(struct vm_map_links)))
38 vm_map_store_init_rb( struct vm_map_header
* hdr
)
40 RB_INIT(&(hdr
->rb_head_store
));
44 rb_node_compare(struct vm_map_store
*node
, struct vm_map_store
*parent
)
49 vme_c
= VME_FOR_STORE(node
);
50 vme_p
= VME_FOR_STORE(parent
);
51 if (vme_c
->vme_start
< vme_p
->vme_start
) {
54 if (vme_c
->vme_start
>= vme_p
->vme_end
) {
62 vm_map_store_walk_rb(vm_map_t map
, vm_map_entry_t
*wrong_vme
, vm_map_entry_t
*vm_entry
)
64 struct vm_map_header
*hdr
= &map
->hdr
;
65 struct vm_map_store
*rb_entry
= RB_ROOT(&hdr
->rb_head_store
);
66 vm_map_entry_t cur
= *vm_entry
;
68 rb_entry
= RB_FIND(rb_head
, &hdr
->rb_head_store
, &(cur
->store
));
69 if (rb_entry
== NULL
) {
70 panic("NO SUCH ENTRY %p. Gave back %p", *vm_entry
, *wrong_vme
);
72 panic("Cur: %p, L: %p, R: %p", VME_FOR_STORE(rb_entry
), VME_FOR_STORE(RB_LEFT(rb_entry
, entry
)), VME_FOR_STORE(RB_RIGHT(rb_entry
, entry
)));
78 vm_map_store_lookup_entry_rb(vm_map_t map
, vm_map_offset_t address
, vm_map_entry_t
*vm_entry
)
80 struct vm_map_header
*hdr
= &map
->hdr
;
81 struct vm_map_store
*rb_entry
= RB_ROOT(&hdr
->rb_head_store
);
82 vm_map_entry_t cur
= vm_map_to_entry(map
);
83 vm_map_entry_t prev
= VM_MAP_ENTRY_NULL
;
85 while (rb_entry
!= (struct vm_map_store
*)NULL
) {
86 cur
= VME_FOR_STORE(rb_entry
);
87 if (cur
== VM_MAP_ENTRY_NULL
) {
90 if (address
>= cur
->vme_start
) {
91 if (address
< cur
->vme_end
) {
95 rb_entry
= RB_RIGHT(rb_entry
, entry
);
98 rb_entry
= RB_LEFT(rb_entry
, entry
);
101 if (prev
== VM_MAP_ENTRY_NULL
) {
102 prev
= vm_map_to_entry(map
);
109 vm_map_store_entry_link_rb( struct vm_map_header
*mapHdr
, __unused vm_map_entry_t after_where
, vm_map_entry_t entry
)
111 struct rb_head
*rbh
= &(mapHdr
->rb_head_store
);
112 struct vm_map_store
*store
= &(entry
->store
);
113 struct vm_map_store
*tmp_store
;
114 if ((tmp_store
= RB_INSERT( rb_head
, rbh
, store
)) != NULL
) {
115 panic("VMSEL: INSERT FAILED: 0x%lx, 0x%lx, 0x%lx, 0x%lx", (uintptr_t)entry
->vme_start
, (uintptr_t)entry
->vme_end
,
116 (uintptr_t)(VME_FOR_STORE(tmp_store
))->vme_start
, (uintptr_t)(VME_FOR_STORE(tmp_store
))->vme_end
);
121 vm_map_store_entry_unlink_rb( struct vm_map_header
*mapHdr
, vm_map_entry_t entry
)
123 struct rb_head
*rbh
= &(mapHdr
->rb_head_store
);
124 struct vm_map_store
*rb_entry
;
125 struct vm_map_store
*store
= &(entry
->store
);
127 rb_entry
= RB_FIND( rb_head
, rbh
, store
);
128 if (rb_entry
== NULL
) {
129 panic("NO ENTRY TO DELETE");
131 RB_REMOVE( rb_head
, rbh
, store
);
135 vm_map_store_copy_reset_rb( vm_map_copy_t copy
, vm_map_entry_t entry
, int nentries
)
137 struct vm_map_header
*mapHdr
= &(copy
->cpy_hdr
);
138 struct rb_head
*rbh
= &(mapHdr
->rb_head_store
);
139 struct vm_map_store
*store
;
142 while (entry
!= vm_map_copy_to_entry(copy
) && nentries
> 0) {
143 store
= &(entry
->store
);
144 RB_REMOVE( rb_head
, rbh
, store
);
145 entry
= entry
->vme_next
;
151 extern zone_t vm_map_holes_zone
; /* zone for vm map holes (vm_map_links) structures */
154 vm_map_combine_hole(vm_map_t map
, vm_map_entry_t hole_entry
);
156 vm_map_combine_hole(__unused vm_map_t map
, vm_map_entry_t hole_entry
)
158 vm_map_entry_t middle_hole_entry
, last_hole_entry
;
160 hole_entry
->vme_end
= hole_entry
->vme_next
->vme_end
;
162 middle_hole_entry
= hole_entry
->vme_next
;
163 last_hole_entry
= middle_hole_entry
->vme_next
;
165 assert(last_hole_entry
->vme_prev
== middle_hole_entry
);
166 assert(middle_hole_entry
->vme_end
!= last_hole_entry
->vme_start
);
168 last_hole_entry
->vme_prev
= hole_entry
;
169 hole_entry
->vme_next
= last_hole_entry
;
171 middle_hole_entry
->vme_prev
= NULL
;
172 middle_hole_entry
->vme_next
= NULL
;
174 zfree(vm_map_holes_zone
, middle_hole_entry
);
176 assert(hole_entry
->vme_start
< hole_entry
->vme_end
);
177 assert(last_hole_entry
->vme_start
< last_hole_entry
->vme_end
);
182 vm_map_delete_hole(vm_map_t map
, vm_map_entry_t hole_entry
);
184 vm_map_delete_hole(vm_map_t map
, vm_map_entry_t hole_entry
)
186 if (hole_entry
== CAST_TO_VM_MAP_ENTRY(map
->holes_list
)) {
187 if (hole_entry
->vme_next
== CAST_TO_VM_MAP_ENTRY(map
->holes_list
)) {
188 map
->holes_list
= NULL
;
189 SAVE_HINT_HOLE_WRITE(map
, NULL
);
191 vm_map_entry_t l_next
, l_prev
;
193 l_next
= (vm_map_entry_t
) map
->holes_list
->next
;
194 l_prev
= (vm_map_entry_t
) map
->holes_list
->prev
;
195 map
->holes_list
= (struct vm_map_links
*) l_next
;
197 l_next
->vme_prev
= l_prev
;
198 l_prev
->vme_next
= l_next
;
200 SAVE_HINT_HOLE_WRITE(map
, (struct vm_map_links
*) l_next
);
203 SAVE_HINT_HOLE_WRITE(map
, (struct vm_map_links
*) hole_entry
->vme_prev
);
205 hole_entry
->vme_prev
->vme_next
= hole_entry
->vme_next
;
206 hole_entry
->vme_next
->vme_prev
= hole_entry
->vme_prev
;
209 hole_entry
->vme_next
= NULL
;
210 hole_entry
->vme_prev
= NULL
;
211 zfree(vm_map_holes_zone
, hole_entry
);
221 check_map_sanity(vm_map_t map
, vm_map_entry_t old_hole_entry
)
223 vm_map_entry_t hole_entry
, next_hole_entry
;
224 vm_map_entry_t map_entry
, next_map_entry
;
226 if (map
->holes_list
== NULL
) {
230 hole_entry
= CAST_DOWN(vm_map_entry_t
, map
->holes_list
);
231 next_hole_entry
= hole_entry
->vme_next
;
233 map_entry
= vm_map_first_entry(map
);
234 next_map_entry
= map_entry
->vme_next
;
236 while (map_entry
->vme_start
> hole_entry
->vme_start
) {
237 hole_entry
= next_hole_entry
;
238 next_hole_entry
= hole_entry
->vme_next
;
240 if (hole_entry
== CAST_DOWN(vm_map_entry_t
, map
->holes_list
)) {
245 while (map_entry
!= vm_map_to_entry(map
)) {
246 if (map_entry
->vme_start
>= map
->max_offset
) {
250 if (map_entry
->vme_end
!= map_entry
->vme_next
->vme_start
) {
251 if (map_entry
->vme_next
== vm_map_to_entry(map
)) {
255 if (hole_entry
->vme_start
!= map_entry
->vme_end
) {
256 panic("hole_entry not aligned %p(0x%llx), %p (0x%llx), %p", hole_entry
, (unsigned long long)hole_entry
->vme_start
, map_entry
->vme_next
, (unsigned long long)map_entry
->vme_end
, old_hole_entry
);
257 assert(hole_entry
->vme_start
== map_entry
->vme_end
);
260 if (hole_entry
->vme_end
!= map_entry
->vme_next
->vme_start
) {
261 panic("hole_entry not next aligned %p(0x%llx), %p (0x%llx), %p", hole_entry
, (unsigned long long)hole_entry
->vme_end
, map_entry
->vme_next
, (unsigned long long)map_entry
->vme_next
->vme_start
, old_hole_entry
);
262 assert(hole_entry
->vme_end
== map_entry
->vme_next
->vme_start
);
265 hole_entry
= next_hole_entry
;
266 next_hole_entry
= hole_entry
->vme_next
;
268 if (hole_entry
== CAST_DOWN(vm_map_entry_t
, map
->holes_list
)) {
273 map_entry
= map_entry
->vme_next
;
281 copy_hole_info(vm_map_entry_t hole_entry
, vm_map_entry_t old_hole_entry
)
283 old_hole_entry
->vme_prev
= hole_entry
->vme_prev
;
284 old_hole_entry
->vme_next
= hole_entry
->vme_next
;
285 old_hole_entry
->vme_start
= hole_entry
->vme_start
;
286 old_hole_entry
->vme_end
= hole_entry
->vme_end
;
291 update_holes_on_entry_deletion(vm_map_t map
, vm_map_entry_t old_entry
);
293 update_holes_on_entry_deletion(vm_map_t map
, vm_map_entry_t old_entry
)
296 * Dealing with the deletion of an older entry.
299 vm_map_entry_t hole_entry
, next_hole_entry
;
301 struct vm_map_entry old_hole_entry
;
303 boolean_t create_new_hole
= TRUE
;
305 hole_entry
= CAST_TO_VM_MAP_ENTRY(map
->hole_hint
);
308 if (hole_entry
->vme_end
== old_entry
->vme_start
) {
310 * Found a hole right after above our entry.
313 } else if (hole_entry
->vme_start
== old_entry
->vme_end
) {
314 if (hole_entry
!= CAST_TO_VM_MAP_ENTRY(map
->holes_list
)) {
316 * Found a hole right after below our entry but
317 * make sure we don't erroneously extend backwards.
322 hole_entry
= hole_entry
->vme_prev
;
324 } else if (hole_entry
->vme_start
> old_entry
->vme_end
) {
326 * Useless hint. Start from the top.
329 hole_entry
= CAST_TO_VM_MAP_ENTRY(map
->holes_list
);
332 if (hole_entry
!= CAST_TO_VM_MAP_ENTRY(map
->holes_list
)) {
333 if (hole_entry
->vme_start
> old_entry
->vme_start
) {
334 panic("Hole hint failed: Hole entry start: 0x%llx, entry start: 0x%llx, map hole start: 0x%llx, map hint start: 0x%llx\n",
335 (unsigned long long)hole_entry
->vme_start
,
336 (unsigned long long)old_entry
->vme_start
,
337 (unsigned long long)map
->holes_list
->start
,
338 (unsigned long long)map
->hole_hint
->start
);
340 if (hole_entry
->vme_end
> old_entry
->vme_start
) {
341 panic("Hole hint failed: Hole entry end: 0x%llx, entry start: 0x%llx, map hole start: 0x%llx, map hint start: 0x%llx\n",
342 (unsigned long long)hole_entry
->vme_end
,
343 (unsigned long long)old_entry
->vme_start
,
344 (unsigned long long)map
->holes_list
->start
,
345 (unsigned long long)map
->hole_hint
->start
);
350 next_hole_entry
= hole_entry
->vme_next
;
353 * Hole is right above the entry.
355 if (hole_entry
->vme_end
== old_entry
->vme_start
) {
357 copy_hole_info(hole_entry
, &old_hole_entry
);
361 * Is there another hole right below the entry?
362 * Can we combine holes?
365 if (old_entry
->vme_end
== hole_entry
->vme_next
->vme_start
) {
366 vm_map_combine_hole(map
, hole_entry
);
368 hole_entry
->vme_end
= old_entry
->vme_end
;
370 create_new_hole
= FALSE
;
372 check_map_sanity(map
, &old_hole_entry
);
378 * Hole is right below the entry.
380 if (hole_entry
->vme_start
== old_entry
->vme_end
) {
382 copy_hole_info(hole_entry
, &old_hole_entry
);
385 hole_entry
->vme_start
= old_entry
->vme_start
;
386 create_new_hole
= FALSE
;
389 check_map_sanity(map
, &old_hole_entry
);
395 * Hole is beyond our entry. Let's go back to the last hole
396 * before our entry so we have the right place to link up the
397 * new hole that will be needed.
399 if (hole_entry
->vme_start
> old_entry
->vme_end
) {
401 copy_hole_info(hole_entry
, &old_hole_entry
);
404 if (hole_entry
!= CAST_TO_VM_MAP_ENTRY(map
->holes_list
)) {
405 assert(hole_entry
->vme_start
!= old_entry
->vme_start
);
406 hole_entry
= hole_entry
->vme_prev
;
411 hole_entry
= next_hole_entry
;
413 if (hole_entry
== CAST_TO_VM_MAP_ENTRY(map
->holes_list
)) {
414 hole_entry
= hole_entry
->vme_prev
;
420 if (create_new_hole
) {
421 struct vm_map_links
*new_hole_entry
= NULL
;
422 vm_map_entry_t l_next
, l_prev
;
424 new_hole_entry
= zalloc(vm_map_holes_zone
);
427 * First hole in the map?
429 * A hole that is located above the current first hole in the map?
431 if (map
->holes_list
== NULL
|| (hole_entry
== CAST_TO_VM_MAP_ENTRY(map
->holes_list
) && hole_entry
->vme_start
> old_entry
->vme_start
)) {
432 if (map
->holes_list
== NULL
) {
433 map
->holes_list
= new_hole_entry
;
434 new_hole_entry
->prev
= new_hole_entry
->next
= CAST_TO_VM_MAP_ENTRY(map
->holes_list
);
436 l_next
= CAST_TO_VM_MAP_ENTRY(map
->holes_list
);
437 l_prev
= map
->holes_list
->prev
;
438 map
->holes_list
= new_hole_entry
;
439 new_hole_entry
->next
= l_next
;
440 new_hole_entry
->prev
= l_prev
;
442 l_prev
->vme_next
= l_next
->vme_prev
= CAST_TO_VM_MAP_ENTRY(new_hole_entry
);
445 l_next
= hole_entry
->vme_next
;
446 l_prev
= hole_entry
->vme_next
->vme_prev
;
448 new_hole_entry
->prev
= hole_entry
;
449 new_hole_entry
->next
= l_next
;
451 hole_entry
->vme_next
= CAST_TO_VM_MAP_ENTRY(new_hole_entry
);
452 l_next
->vme_prev
= CAST_TO_VM_MAP_ENTRY(new_hole_entry
);
455 new_hole_entry
->start
= old_entry
->vme_start
;
456 new_hole_entry
->end
= old_entry
->vme_end
;
458 hole_entry
= CAST_TO_VM_MAP_ENTRY(new_hole_entry
);
460 assert(new_hole_entry
->start
< new_hole_entry
->end
);
464 check_map_sanity(map
, &old_hole_entry
);
467 SAVE_HINT_HOLE_WRITE(map
, (struct vm_map_links
*) hole_entry
);
473 update_holes_on_entry_creation(vm_map_t map
, vm_map_entry_t new_entry
);
475 update_holes_on_entry_creation(vm_map_t map
, vm_map_entry_t new_entry
)
477 vm_map_entry_t hole_entry
, next_hole_entry
;
479 struct vm_map_entry old_hole_entry
;
480 vm_map_entry_t tmp_entry
;
481 boolean_t check_map_with_hole_sanity
= TRUE
;
485 * Case A: The entry is aligned exactly with the start and end of the hole.
486 * This will delete the hole.
488 * Case B: The entry is completely within a hole but NOT aligned with the start/end of the hole.
489 * This will split a hole.
491 * Case C: The entry overlaps with the hole. The entry could be extending upwards (C1) or downwards (C2).
492 * This will reduce the size of the hole or delete the hole completely if it is smaller than the entry.
495 hole_entry
= CAST_TO_VM_MAP_ENTRY(map
->holes_list
);
497 next_hole_entry
= hole_entry
->vme_next
;
502 * If the entry doesn't exist in the RB tree, we are likely dealing with copy maps where
503 * the entries belonging to the copy map are linked into the list of entries silently and
504 * then added to the RB-tree later on.
505 * So sanity checks are useless in that case.
507 check_map_with_hole_sanity
= vm_map_lookup_entry(map
, new_entry
->vme_start
, &tmp_entry
);
510 if (hole_entry
->vme_start
== new_entry
->vme_start
&&
511 hole_entry
->vme_end
== new_entry
->vme_end
) {
514 copy_hole_info(hole_entry
, &old_hole_entry
);
518 * This check makes sense only for regular maps, not copy maps.
519 * With a regular map, the VM entry is first linked and then
520 * the hole is deleted. So the check below, which makes sure that
521 * the map's bounds are being respected, is valid.
522 * But for copy maps, the hole is deleted before the VM entry is
523 * linked (vm_map_store_copy_insert) and so this check is invalid.
525 * if (hole_entry == (vm_map_entry_t) map->holes_list) {
527 * if (hole_entry->vme_next == (vm_map_entry_t) map->holes_list) {
529 * next_hole_entry = vm_map_last_entry(map);
530 * assert(next_hole_entry->vme_end >= map->max_offset);
535 vm_map_delete_hole(map
, hole_entry
);
538 if (check_map_with_hole_sanity
) {
539 check_map_sanity(map
, &old_hole_entry
);
543 } else if (hole_entry
->vme_start
< new_entry
->vme_start
&&
544 hole_entry
->vme_end
> new_entry
->vme_end
) {
546 struct vm_map_links
*new_hole_entry
= NULL
;
548 new_hole_entry
= zalloc(vm_map_holes_zone
);
551 copy_hole_info(hole_entry
, &old_hole_entry
);
554 new_hole_entry
->prev
= hole_entry
;
555 new_hole_entry
->next
= hole_entry
->vme_next
;
556 hole_entry
->vme_next
->vme_prev
= CAST_TO_VM_MAP_ENTRY(new_hole_entry
);
557 hole_entry
->vme_next
= CAST_TO_VM_MAP_ENTRY(new_hole_entry
);
559 new_hole_entry
->start
= new_entry
->vme_end
;
560 new_hole_entry
->end
= hole_entry
->vme_end
;
561 hole_entry
->vme_end
= new_entry
->vme_start
;
563 assert(hole_entry
->vme_start
< hole_entry
->vme_end
);
564 assert(new_hole_entry
->start
< new_hole_entry
->end
);
567 if (check_map_with_hole_sanity
) {
568 check_map_sanity(map
, &old_hole_entry
);
572 SAVE_HINT_HOLE_WRITE(map
, (struct vm_map_links
*) hole_entry
);
574 } else if ((new_entry
->vme_start
<= hole_entry
->vme_start
) && (hole_entry
->vme_start
< new_entry
->vme_end
)) {
576 * Case C1: Entry moving upwards and a part/full hole lies within the bounds of the entry.
580 copy_hole_info(hole_entry
, &old_hole_entry
);
583 if (hole_entry
->vme_end
<= new_entry
->vme_end
) {
584 vm_map_delete_hole(map
, hole_entry
);
586 hole_entry
->vme_start
= new_entry
->vme_end
;
587 SAVE_HINT_HOLE_WRITE(map
, (struct vm_map_links
*) hole_entry
);
591 if (check_map_with_hole_sanity
) {
592 check_map_sanity(map
, &old_hole_entry
);
597 } else if ((new_entry
->vme_start
< hole_entry
->vme_end
) && (hole_entry
->vme_end
<= new_entry
->vme_end
)) {
599 * Case C2: Entry moving downwards and a part/full hole lies within the bounds of the entry.
603 copy_hole_info(hole_entry
, &old_hole_entry
);
606 if (hole_entry
->vme_start
>= new_entry
->vme_start
) {
607 vm_map_delete_hole(map
, hole_entry
);
609 hole_entry
->vme_end
= new_entry
->vme_start
;
610 SAVE_HINT_HOLE_WRITE(map
, (struct vm_map_links
*) hole_entry
);
614 if (check_map_with_hole_sanity
) {
615 check_map_sanity(map
, &old_hole_entry
);
622 hole_entry
= next_hole_entry
;
623 next_hole_entry
= hole_entry
->vme_next
;
625 if (hole_entry
== CAST_TO_VM_MAP_ENTRY(map
->holes_list
)) {
630 panic("Illegal action: h1: %p, s:0x%llx, e:0x%llx...h2:%p, s:0x%llx, e:0x%llx...h3:0x%p, s:0x%llx, e:0x%llx\n",
631 hole_entry
->vme_prev
,
632 (unsigned long long)hole_entry
->vme_prev
->vme_start
,
633 (unsigned long long)hole_entry
->vme_prev
->vme_end
,
635 (unsigned long long)hole_entry
->vme_start
,
636 (unsigned long long)hole_entry
->vme_end
,
637 hole_entry
->vme_next
,
638 (unsigned long long)hole_entry
->vme_next
->vme_start
,
639 (unsigned long long)hole_entry
->vme_next
->vme_end
);
643 update_first_free_rb(vm_map_t map
, vm_map_entry_t entry
, boolean_t new_entry_creation
)
645 if (map
->holelistenabled
) {
647 * Holes can be used to track ranges all the way up to MACH_VM_MAX_ADDRESS or more (e.g. kernel map).
649 vm_map_offset_t max_valid_offset
= (map
->max_offset
> MACH_VM_MAX_ADDRESS
) ? map
->max_offset
: MACH_VM_MAX_ADDRESS
;
652 * Clipping an entry will not result in the creation/deletion/modification of
653 * a hole. Those calls pass NULL for their target entry.
660 * Commpage is pinned beyond the map's max offset. That shouldn't affect the
661 * holes within the bounds of the map.
663 if (vm_map_trunc_page(entry
->vme_start
, VM_MAP_PAGE_MASK(map
)) >= max_valid_offset
) {
671 * - A new entry has already been added to the map
673 * - An older entry has already been deleted from the map
675 * We are updating the hole list after the fact (except in one special case involving copy maps).
679 if (new_entry_creation
) {
680 update_holes_on_entry_creation(map
, entry
);
682 update_holes_on_entry_deletion(map
, entry
);