]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_fault.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-6153.141.1.tar.gz
[apple/xnu.git] / osfmk / vm / vm_fault.c
1 /*
2 * Copyright (c) 2000-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm_fault.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * Page fault handling module.
63 */
64
65 #include <mach_cluster_stats.h>
66 #include <mach_pagemap.h>
67 #include <libkern/OSAtomic.h>
68
69 #include <mach/mach_types.h>
70 #include <mach/kern_return.h>
71 #include <mach/message.h> /* for error codes */
72 #include <mach/vm_param.h>
73 #include <mach/vm_behavior.h>
74 #include <mach/memory_object.h>
75 /* For memory_object_data_{request,unlock} */
76 #include <mach/sdt.h>
77
78 #include <kern/kern_types.h>
79 #include <kern/host_statistics.h>
80 #include <kern/counters.h>
81 #include <kern/task.h>
82 #include <kern/thread.h>
83 #include <kern/sched_prim.h>
84 #include <kern/host.h>
85 #include <kern/mach_param.h>
86 #include <kern/macro_help.h>
87 #include <kern/zalloc.h>
88 #include <kern/misc_protos.h>
89 #include <kern/policy_internal.h>
90
91 #include <vm/vm_compressor.h>
92 #include <vm/vm_compressor_pager.h>
93 #include <vm/vm_fault.h>
94 #include <vm/vm_map.h>
95 #include <vm/vm_object.h>
96 #include <vm/vm_page.h>
97 #include <vm/vm_kern.h>
98 #include <vm/pmap.h>
99 #include <vm/vm_pageout.h>
100 #include <vm/vm_protos.h>
101 #include <vm/vm_external.h>
102 #include <vm/memory_object.h>
103 #include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */
104 #include <vm/vm_shared_region.h>
105
106 #include <sys/codesign.h>
107 #include <sys/reason.h>
108 #include <sys/signalvar.h>
109
110 #include <san/kasan.h>
111
112 #define VM_FAULT_CLASSIFY 0
113
114 #define TRACEFAULTPAGE 0 /* (TEST/DEBUG) */
115
116 int vm_protect_privileged_from_untrusted = 1;
117
118 unsigned int vm_object_pagein_throttle = 16;
119
120 /*
121 * We apply a hard throttle to the demand zero rate of tasks that we believe are running out of control which
122 * kicks in when swap space runs out. 64-bit programs have massive address spaces and can leak enormous amounts
123 * of memory if they're buggy and can run the system completely out of swap space. If this happens, we
124 * impose a hard throttle on them to prevent them from taking the last bit of memory left. This helps
125 * keep the UI active so that the user has a chance to kill the offending task before the system
126 * completely hangs.
127 *
128 * The hard throttle is only applied when the system is nearly completely out of swap space and is only applied
129 * to tasks that appear to be bloated. When swap runs out, any task using more than vm_hard_throttle_threshold
130 * will be throttled. The throttling is done by giving the thread that's trying to demand zero a page a
131 * delay of HARD_THROTTLE_DELAY microseconds before being allowed to try the page fault again.
132 */
133
134 extern void throttle_lowpri_io(int);
135
136 extern struct vnode *vnode_pager_lookup_vnode(memory_object_t);
137
138 uint64_t vm_hard_throttle_threshold;
139
140
141 OS_ALWAYS_INLINE
142 boolean_t
143 NEED_TO_HARD_THROTTLE_THIS_TASK(void)
144 {
145 return vm_wants_task_throttled(current_task()) ||
146 ((vm_page_free_count < vm_page_throttle_limit ||
147 HARD_THROTTLE_LIMIT_REACHED()) &&
148 proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO) >= THROTTLE_LEVEL_THROTTLED);
149 }
150
151 #define HARD_THROTTLE_DELAY 10000 /* 10000 us == 10 ms */
152 #define SOFT_THROTTLE_DELAY 200 /* 200 us == .2 ms */
153
154 #define VM_PAGE_CREATION_THROTTLE_PERIOD_SECS 6
155 #define VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC 20000
156
157
158 #define VM_STAT_DECOMPRESSIONS() \
159 MACRO_BEGIN \
160 VM_STAT_INCR(decompressions); \
161 current_thread()->decompressions++; \
162 MACRO_END
163
164 boolean_t current_thread_aborted(void);
165
166 /* Forward declarations of internal routines. */
167 static kern_return_t vm_fault_wire_fast(
168 vm_map_t map,
169 vm_map_offset_t va,
170 vm_prot_t prot,
171 vm_tag_t wire_tag,
172 vm_map_entry_t entry,
173 pmap_t pmap,
174 vm_map_offset_t pmap_addr,
175 ppnum_t *physpage_p);
176
177 static kern_return_t vm_fault_internal(
178 vm_map_t map,
179 vm_map_offset_t vaddr,
180 vm_prot_t caller_prot,
181 boolean_t change_wiring,
182 vm_tag_t wire_tag,
183 int interruptible,
184 pmap_t pmap,
185 vm_map_offset_t pmap_addr,
186 ppnum_t *physpage_p);
187
188 static void vm_fault_copy_cleanup(
189 vm_page_t page,
190 vm_page_t top_page);
191
192 static void vm_fault_copy_dst_cleanup(
193 vm_page_t page);
194
195 #if VM_FAULT_CLASSIFY
196 extern void vm_fault_classify(vm_object_t object,
197 vm_object_offset_t offset,
198 vm_prot_t fault_type);
199
200 extern void vm_fault_classify_init(void);
201 #endif
202
203 unsigned long vm_pmap_enter_blocked = 0;
204 unsigned long vm_pmap_enter_retried = 0;
205
206 unsigned long vm_cs_validates = 0;
207 unsigned long vm_cs_revalidates = 0;
208 unsigned long vm_cs_query_modified = 0;
209 unsigned long vm_cs_validated_dirtied = 0;
210 unsigned long vm_cs_bitmap_validated = 0;
211 #if PMAP_CS
212 uint64_t vm_cs_defer_to_pmap_cs = 0;
213 uint64_t vm_cs_defer_to_pmap_cs_not = 0;
214 #endif /* PMAP_CS */
215
216 void vm_pre_fault(vm_map_offset_t, vm_prot_t);
217
218 extern char *kdp_compressor_decompressed_page;
219 extern addr64_t kdp_compressor_decompressed_page_paddr;
220 extern ppnum_t kdp_compressor_decompressed_page_ppnum;
221
222 struct vmrtfr {
223 int vmrtfr_maxi;
224 int vmrtfr_curi;
225 int64_t vmrtf_total;
226 vm_rtfault_record_t *vm_rtf_records;
227 } vmrtfrs;
228 #define VMRTF_DEFAULT_BUFSIZE (4096)
229 #define VMRTF_NUM_RECORDS_DEFAULT (VMRTF_DEFAULT_BUFSIZE / sizeof(vm_rtfault_record_t))
230 int vmrtf_num_records = VMRTF_NUM_RECORDS_DEFAULT;
231
232 static void vm_rtfrecord_lock(void);
233 static void vm_rtfrecord_unlock(void);
234 static void vm_record_rtfault(thread_t, uint64_t, vm_map_offset_t, int);
235
236 lck_spin_t vm_rtfr_slock;
237 extern lck_grp_t vm_page_lck_grp_bucket;
238 extern lck_attr_t vm_page_lck_attr;
239
240 /*
241 * Routine: vm_fault_init
242 * Purpose:
243 * Initialize our private data structures.
244 */
245 void
246 vm_fault_init(void)
247 {
248 int i, vm_compressor_temp;
249 boolean_t need_default_val = TRUE;
250 /*
251 * Choose a value for the hard throttle threshold based on the amount of ram. The threshold is
252 * computed as a percentage of available memory, and the percentage used is scaled inversely with
253 * the amount of memory. The percentage runs between 10% and 35%. We use 35% for small memory systems
254 * and reduce the value down to 10% for very large memory configurations. This helps give us a
255 * definition of a memory hog that makes more sense relative to the amount of ram in the machine.
256 * The formula here simply uses the number of gigabytes of ram to adjust the percentage.
257 */
258
259 vm_hard_throttle_threshold = sane_size * (35 - MIN((int)(sane_size / (1024 * 1024 * 1024)), 25)) / 100;
260
261 /*
262 * Configure compressed pager behavior. A boot arg takes precedence over a device tree entry.
263 */
264
265 if (PE_parse_boot_argn("vm_compressor", &vm_compressor_temp, sizeof(vm_compressor_temp))) {
266 for (i = 0; i < VM_PAGER_MAX_MODES; i++) {
267 if (vm_compressor_temp > 0 &&
268 ((vm_compressor_temp & (1 << i)) == vm_compressor_temp)) {
269 need_default_val = FALSE;
270 vm_compressor_mode = vm_compressor_temp;
271 break;
272 }
273 }
274 if (need_default_val) {
275 printf("Ignoring \"vm_compressor\" boot arg %d\n", vm_compressor_temp);
276 }
277 }
278 if (need_default_val) {
279 /* If no boot arg or incorrect boot arg, try device tree. */
280 PE_get_default("kern.vm_compressor", &vm_compressor_mode, sizeof(vm_compressor_mode));
281 }
282 printf("\"vm_compressor_mode\" is %d\n", vm_compressor_mode);
283
284 PE_parse_boot_argn("vm_protect_privileged_from_untrusted", &vm_protect_privileged_from_untrusted, sizeof(vm_protect_privileged_from_untrusted));
285 }
286
287 void
288 vm_rtfault_record_init(void)
289 {
290 PE_parse_boot_argn("vm_rtfault_records", &vmrtf_num_records, sizeof(vmrtf_num_records));
291
292 assert(vmrtf_num_records >= 1);
293 vmrtf_num_records = MAX(vmrtf_num_records, 1);
294 size_t kallocsz = vmrtf_num_records * sizeof(vm_rtfault_record_t);
295 vmrtfrs.vm_rtf_records = kalloc(kallocsz);
296 bzero(vmrtfrs.vm_rtf_records, kallocsz);
297 vmrtfrs.vmrtfr_maxi = vmrtf_num_records - 1;
298 lck_spin_init(&vm_rtfr_slock, &vm_page_lck_grp_bucket, &vm_page_lck_attr);
299 }
300 /*
301 * Routine: vm_fault_cleanup
302 * Purpose:
303 * Clean up the result of vm_fault_page.
304 * Results:
305 * The paging reference for "object" is released.
306 * "object" is unlocked.
307 * If "top_page" is not null, "top_page" is
308 * freed and the paging reference for the object
309 * containing it is released.
310 *
311 * In/out conditions:
312 * "object" must be locked.
313 */
314 void
315 vm_fault_cleanup(
316 vm_object_t object,
317 vm_page_t top_page)
318 {
319 vm_object_paging_end(object);
320 vm_object_unlock(object);
321
322 if (top_page != VM_PAGE_NULL) {
323 object = VM_PAGE_OBJECT(top_page);
324
325 vm_object_lock(object);
326 VM_PAGE_FREE(top_page);
327 vm_object_paging_end(object);
328 vm_object_unlock(object);
329 }
330 }
331
332 #define ALIGNED(x) (((x) & (PAGE_SIZE_64 - 1)) == 0)
333
334
335 boolean_t vm_page_deactivate_behind = TRUE;
336 /*
337 * default sizes given VM_BEHAVIOR_DEFAULT reference behavior
338 */
339 #define VM_DEFAULT_DEACTIVATE_BEHIND_WINDOW 128
340 #define VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER 16 /* don't make this too big... */
341 /* we use it to size an array on the stack */
342
343 int vm_default_behind = VM_DEFAULT_DEACTIVATE_BEHIND_WINDOW;
344
345 #define MAX_SEQUENTIAL_RUN (1024 * 1024 * 1024)
346
347 /*
348 * vm_page_is_sequential
349 *
350 * Determine if sequential access is in progress
351 * in accordance with the behavior specified.
352 * Update state to indicate current access pattern.
353 *
354 * object must have at least the shared lock held
355 */
356 static
357 void
358 vm_fault_is_sequential(
359 vm_object_t object,
360 vm_object_offset_t offset,
361 vm_behavior_t behavior)
362 {
363 vm_object_offset_t last_alloc;
364 int sequential;
365 int orig_sequential;
366
367 last_alloc = object->last_alloc;
368 sequential = object->sequential;
369 orig_sequential = sequential;
370
371 switch (behavior) {
372 case VM_BEHAVIOR_RANDOM:
373 /*
374 * reset indicator of sequential behavior
375 */
376 sequential = 0;
377 break;
378
379 case VM_BEHAVIOR_SEQUENTIAL:
380 if (offset && last_alloc == offset - PAGE_SIZE_64) {
381 /*
382 * advance indicator of sequential behavior
383 */
384 if (sequential < MAX_SEQUENTIAL_RUN) {
385 sequential += PAGE_SIZE;
386 }
387 } else {
388 /*
389 * reset indicator of sequential behavior
390 */
391 sequential = 0;
392 }
393 break;
394
395 case VM_BEHAVIOR_RSEQNTL:
396 if (last_alloc && last_alloc == offset + PAGE_SIZE_64) {
397 /*
398 * advance indicator of sequential behavior
399 */
400 if (sequential > -MAX_SEQUENTIAL_RUN) {
401 sequential -= PAGE_SIZE;
402 }
403 } else {
404 /*
405 * reset indicator of sequential behavior
406 */
407 sequential = 0;
408 }
409 break;
410
411 case VM_BEHAVIOR_DEFAULT:
412 default:
413 if (offset && last_alloc == (offset - PAGE_SIZE_64)) {
414 /*
415 * advance indicator of sequential behavior
416 */
417 if (sequential < 0) {
418 sequential = 0;
419 }
420 if (sequential < MAX_SEQUENTIAL_RUN) {
421 sequential += PAGE_SIZE;
422 }
423 } else if (last_alloc && last_alloc == (offset + PAGE_SIZE_64)) {
424 /*
425 * advance indicator of sequential behavior
426 */
427 if (sequential > 0) {
428 sequential = 0;
429 }
430 if (sequential > -MAX_SEQUENTIAL_RUN) {
431 sequential -= PAGE_SIZE;
432 }
433 } else {
434 /*
435 * reset indicator of sequential behavior
436 */
437 sequential = 0;
438 }
439 break;
440 }
441 if (sequential != orig_sequential) {
442 if (!OSCompareAndSwap(orig_sequential, sequential, (UInt32 *)&object->sequential)) {
443 /*
444 * if someone else has already updated object->sequential
445 * don't bother trying to update it or object->last_alloc
446 */
447 return;
448 }
449 }
450 /*
451 * I'd like to do this with a OSCompareAndSwap64, but that
452 * doesn't exist for PPC... however, it shouldn't matter
453 * that much... last_alloc is maintained so that we can determine
454 * if a sequential access pattern is taking place... if only
455 * one thread is banging on this object, no problem with the unprotected
456 * update... if 2 or more threads are banging away, we run the risk of
457 * someone seeing a mangled update... however, in the face of multiple
458 * accesses, no sequential access pattern can develop anyway, so we
459 * haven't lost any real info.
460 */
461 object->last_alloc = offset;
462 }
463
464
465 int vm_page_deactivate_behind_count = 0;
466
467 /*
468 * vm_page_deactivate_behind
469 *
470 * Determine if sequential access is in progress
471 * in accordance with the behavior specified. If
472 * so, compute a potential page to deactivate and
473 * deactivate it.
474 *
475 * object must be locked.
476 *
477 * return TRUE if we actually deactivate a page
478 */
479 static
480 boolean_t
481 vm_fault_deactivate_behind(
482 vm_object_t object,
483 vm_object_offset_t offset,
484 vm_behavior_t behavior)
485 {
486 int n;
487 int pages_in_run = 0;
488 int max_pages_in_run = 0;
489 int sequential_run;
490 int sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
491 vm_object_offset_t run_offset = 0;
492 vm_object_offset_t pg_offset = 0;
493 vm_page_t m;
494 vm_page_t page_run[VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER];
495
496 pages_in_run = 0;
497 #if TRACEFAULTPAGE
498 dbgTrace(0xBEEF0018, (unsigned int) object, (unsigned int) vm_fault_deactivate_behind); /* (TEST/DEBUG) */
499 #endif
500
501 if (object == kernel_object || vm_page_deactivate_behind == FALSE) {
502 /*
503 * Do not deactivate pages from the kernel object: they
504 * are not intended to become pageable.
505 * or we've disabled the deactivate behind mechanism
506 */
507 return FALSE;
508 }
509 if ((sequential_run = object->sequential)) {
510 if (sequential_run < 0) {
511 sequential_behavior = VM_BEHAVIOR_RSEQNTL;
512 sequential_run = 0 - sequential_run;
513 } else {
514 sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
515 }
516 }
517 switch (behavior) {
518 case VM_BEHAVIOR_RANDOM:
519 break;
520 case VM_BEHAVIOR_SEQUENTIAL:
521 if (sequential_run >= (int)PAGE_SIZE) {
522 run_offset = 0 - PAGE_SIZE_64;
523 max_pages_in_run = 1;
524 }
525 break;
526 case VM_BEHAVIOR_RSEQNTL:
527 if (sequential_run >= (int)PAGE_SIZE) {
528 run_offset = PAGE_SIZE_64;
529 max_pages_in_run = 1;
530 }
531 break;
532 case VM_BEHAVIOR_DEFAULT:
533 default:
534 { vm_object_offset_t behind = vm_default_behind * PAGE_SIZE_64;
535
536 /*
537 * determine if the run of sequential accesss has been
538 * long enough on an object with default access behavior
539 * to consider it for deactivation
540 */
541 if ((uint64_t)sequential_run >= behind && (sequential_run % (VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER * PAGE_SIZE)) == 0) {
542 /*
543 * the comparisons between offset and behind are done
544 * in this kind of odd fashion in order to prevent wrap around
545 * at the end points
546 */
547 if (sequential_behavior == VM_BEHAVIOR_SEQUENTIAL) {
548 if (offset >= behind) {
549 run_offset = 0 - behind;
550 pg_offset = PAGE_SIZE_64;
551 max_pages_in_run = VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER;
552 }
553 } else {
554 if (offset < -behind) {
555 run_offset = behind;
556 pg_offset = 0 - PAGE_SIZE_64;
557 max_pages_in_run = VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER;
558 }
559 }
560 }
561 break;}
562 }
563 for (n = 0; n < max_pages_in_run; n++) {
564 m = vm_page_lookup(object, offset + run_offset + (n * pg_offset));
565
566 if (m && !m->vmp_laundry && !m->vmp_busy && !m->vmp_no_cache && (m->vmp_q_state != VM_PAGE_ON_THROTTLED_Q) && !m->vmp_fictitious && !m->vmp_absent) {
567 page_run[pages_in_run++] = m;
568
569 /*
570 * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise...
571 *
572 * a TLB flush isn't really needed here since at worst we'll miss the reference bit being
573 * updated in the PTE if a remote processor still has this mapping cached in its TLB when the
574 * new reference happens. If no futher references happen on the page after that remote TLB flushes
575 * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue
576 * by pageout_scan, which is just fine since the last reference would have happened quite far
577 * in the past (TLB caches don't hang around for very long), and of course could just as easily
578 * have happened before we did the deactivate_behind.
579 */
580 pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
581 }
582 }
583 if (pages_in_run) {
584 vm_page_lockspin_queues();
585
586 for (n = 0; n < pages_in_run; n++) {
587 m = page_run[n];
588
589 vm_page_deactivate_internal(m, FALSE);
590
591 vm_page_deactivate_behind_count++;
592 #if TRACEFAULTPAGE
593 dbgTrace(0xBEEF0019, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
594 #endif
595 }
596 vm_page_unlock_queues();
597
598 return TRUE;
599 }
600 return FALSE;
601 }
602
603
604 #if (DEVELOPMENT || DEBUG)
605 uint32_t vm_page_creation_throttled_hard = 0;
606 uint32_t vm_page_creation_throttled_soft = 0;
607 uint64_t vm_page_creation_throttle_avoided = 0;
608 #endif /* DEVELOPMENT || DEBUG */
609
610 static int
611 vm_page_throttled(boolean_t page_kept)
612 {
613 clock_sec_t elapsed_sec;
614 clock_sec_t tv_sec;
615 clock_usec_t tv_usec;
616
617 thread_t thread = current_thread();
618
619 if (thread->options & TH_OPT_VMPRIV) {
620 return 0;
621 }
622
623 if (thread->t_page_creation_throttled) {
624 thread->t_page_creation_throttled = 0;
625
626 if (page_kept == FALSE) {
627 goto no_throttle;
628 }
629 }
630 if (NEED_TO_HARD_THROTTLE_THIS_TASK()) {
631 #if (DEVELOPMENT || DEBUG)
632 thread->t_page_creation_throttled_hard++;
633 OSAddAtomic(1, &vm_page_creation_throttled_hard);
634 #endif /* DEVELOPMENT || DEBUG */
635 return HARD_THROTTLE_DELAY;
636 }
637
638 if ((vm_page_free_count < vm_page_throttle_limit || (VM_CONFIG_COMPRESSOR_IS_PRESENT && SWAPPER_NEEDS_TO_UNTHROTTLE())) &&
639 thread->t_page_creation_count > (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS * VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC)) {
640 if (vm_page_free_wanted == 0 && vm_page_free_wanted_privileged == 0) {
641 #if (DEVELOPMENT || DEBUG)
642 OSAddAtomic64(1, &vm_page_creation_throttle_avoided);
643 #endif
644 goto no_throttle;
645 }
646 clock_get_system_microtime(&tv_sec, &tv_usec);
647
648 elapsed_sec = tv_sec - thread->t_page_creation_time;
649
650 if (elapsed_sec <= VM_PAGE_CREATION_THROTTLE_PERIOD_SECS ||
651 (thread->t_page_creation_count / elapsed_sec) >= VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC) {
652 if (elapsed_sec >= (3 * VM_PAGE_CREATION_THROTTLE_PERIOD_SECS)) {
653 /*
654 * we'll reset our stats to give a well behaved app
655 * that was unlucky enough to accumulate a bunch of pages
656 * over a long period of time a chance to get out of
657 * the throttled state... we reset the counter and timestamp
658 * so that if it stays under the rate limit for the next second
659 * it will be back in our good graces... if it exceeds it, it
660 * will remain in the throttled state
661 */
662 thread->t_page_creation_time = tv_sec;
663 thread->t_page_creation_count = VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC * (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS - 1);
664 }
665 VM_PAGEOUT_DEBUG(vm_page_throttle_count, 1);
666
667 thread->t_page_creation_throttled = 1;
668
669 if (VM_CONFIG_COMPRESSOR_IS_PRESENT && HARD_THROTTLE_LIMIT_REACHED()) {
670 #if (DEVELOPMENT || DEBUG)
671 thread->t_page_creation_throttled_hard++;
672 OSAddAtomic(1, &vm_page_creation_throttled_hard);
673 #endif /* DEVELOPMENT || DEBUG */
674 return HARD_THROTTLE_DELAY;
675 } else {
676 #if (DEVELOPMENT || DEBUG)
677 thread->t_page_creation_throttled_soft++;
678 OSAddAtomic(1, &vm_page_creation_throttled_soft);
679 #endif /* DEVELOPMENT || DEBUG */
680 return SOFT_THROTTLE_DELAY;
681 }
682 }
683 thread->t_page_creation_time = tv_sec;
684 thread->t_page_creation_count = 0;
685 }
686 no_throttle:
687 thread->t_page_creation_count++;
688
689 return 0;
690 }
691
692
693 /*
694 * check for various conditions that would
695 * prevent us from creating a ZF page...
696 * cleanup is based on being called from vm_fault_page
697 *
698 * object must be locked
699 * object == m->vmp_object
700 */
701 static vm_fault_return_t
702 vm_fault_check(vm_object_t object, vm_page_t m, vm_page_t first_m, wait_interrupt_t interruptible_state, boolean_t page_throttle)
703 {
704 int throttle_delay;
705
706 if (object->shadow_severed ||
707 VM_OBJECT_PURGEABLE_FAULT_ERROR(object)) {
708 /*
709 * Either:
710 * 1. the shadow chain was severed,
711 * 2. the purgeable object is volatile or empty and is marked
712 * to fault on access while volatile.
713 * Just have to return an error at this point
714 */
715 if (m != VM_PAGE_NULL) {
716 VM_PAGE_FREE(m);
717 }
718 vm_fault_cleanup(object, first_m);
719
720 thread_interrupt_level(interruptible_state);
721
722 return VM_FAULT_MEMORY_ERROR;
723 }
724 if (page_throttle == TRUE) {
725 if ((throttle_delay = vm_page_throttled(FALSE))) {
726 /*
727 * we're throttling zero-fills...
728 * treat this as if we couldn't grab a page
729 */
730 if (m != VM_PAGE_NULL) {
731 VM_PAGE_FREE(m);
732 }
733 vm_fault_cleanup(object, first_m);
734
735 VM_DEBUG_EVENT(vmf_check_zfdelay, VMF_CHECK_ZFDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
736
737 delay(throttle_delay);
738
739 if (current_thread_aborted()) {
740 thread_interrupt_level(interruptible_state);
741 return VM_FAULT_INTERRUPTED;
742 }
743 thread_interrupt_level(interruptible_state);
744
745 return VM_FAULT_MEMORY_SHORTAGE;
746 }
747 }
748 return VM_FAULT_SUCCESS;
749 }
750
751
752 /*
753 * do the work to zero fill a page and
754 * inject it into the correct paging queue
755 *
756 * m->vmp_object must be locked
757 * page queue lock must NOT be held
758 */
759 static int
760 vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill)
761 {
762 int my_fault = DBG_ZERO_FILL_FAULT;
763 vm_object_t object;
764
765 object = VM_PAGE_OBJECT(m);
766
767 /*
768 * This is is a zero-fill page fault...
769 *
770 * Checking the page lock is a waste of
771 * time; this page was absent, so
772 * it can't be page locked by a pager.
773 *
774 * we also consider it undefined
775 * with respect to instruction
776 * execution. i.e. it is the responsibility
777 * of higher layers to call for an instruction
778 * sync after changing the contents and before
779 * sending a program into this area. We
780 * choose this approach for performance
781 */
782 m->vmp_pmapped = TRUE;
783
784 m->vmp_cs_validated = FALSE;
785 m->vmp_cs_tainted = FALSE;
786 m->vmp_cs_nx = FALSE;
787
788 if (no_zero_fill == TRUE) {
789 my_fault = DBG_NZF_PAGE_FAULT;
790
791 if (m->vmp_absent && m->vmp_busy) {
792 return my_fault;
793 }
794 } else {
795 vm_page_zero_fill(m);
796
797 VM_STAT_INCR(zero_fill_count);
798 DTRACE_VM2(zfod, int, 1, (uint64_t *), NULL);
799 }
800 assert(!m->vmp_laundry);
801 assert(object != kernel_object);
802 //assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
803
804 if (!VM_DYNAMIC_PAGING_ENABLED() &&
805 (object->purgable == VM_PURGABLE_DENY ||
806 object->purgable == VM_PURGABLE_NONVOLATILE ||
807 object->purgable == VM_PURGABLE_VOLATILE)) {
808 vm_page_lockspin_queues();
809
810 if (!VM_DYNAMIC_PAGING_ENABLED()) {
811 assert(!VM_PAGE_WIRED(m));
812
813 /*
814 * can't be on the pageout queue since we don't
815 * have a pager to try and clean to
816 */
817 vm_page_queues_remove(m, TRUE);
818 vm_page_check_pageable_safe(m);
819 vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);
820 m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
821 vm_page_throttled_count++;
822 }
823 vm_page_unlock_queues();
824 }
825 return my_fault;
826 }
827
828
829 /*
830 * Routine: vm_fault_page
831 * Purpose:
832 * Find the resident page for the virtual memory
833 * specified by the given virtual memory object
834 * and offset.
835 * Additional arguments:
836 * The required permissions for the page is given
837 * in "fault_type". Desired permissions are included
838 * in "protection".
839 * fault_info is passed along to determine pagein cluster
840 * limits... it contains the expected reference pattern,
841 * cluster size if available, etc...
842 *
843 * If the desired page is known to be resident (for
844 * example, because it was previously wired down), asserting
845 * the "unwiring" parameter will speed the search.
846 *
847 * If the operation can be interrupted (by thread_abort
848 * or thread_terminate), then the "interruptible"
849 * parameter should be asserted.
850 *
851 * Results:
852 * The page containing the proper data is returned
853 * in "result_page".
854 *
855 * In/out conditions:
856 * The source object must be locked and referenced,
857 * and must donate one paging reference. The reference
858 * is not affected. The paging reference and lock are
859 * consumed.
860 *
861 * If the call succeeds, the object in which "result_page"
862 * resides is left locked and holding a paging reference.
863 * If this is not the original object, a busy page in the
864 * original object is returned in "top_page", to prevent other
865 * callers from pursuing this same data, along with a paging
866 * reference for the original object. The "top_page" should
867 * be destroyed when this guarantee is no longer required.
868 * The "result_page" is also left busy. It is not removed
869 * from the pageout queues.
870 * Special Case:
871 * A return value of VM_FAULT_SUCCESS_NO_PAGE means that the
872 * fault succeeded but there's no VM page (i.e. the VM object
873 * does not actually hold VM pages, but device memory or
874 * large pages). The object is still locked and we still hold a
875 * paging_in_progress reference.
876 */
877 unsigned int vm_fault_page_blocked_access = 0;
878 unsigned int vm_fault_page_forced_retry = 0;
879
880 vm_fault_return_t
881 vm_fault_page(
882 /* Arguments: */
883 vm_object_t first_object, /* Object to begin search */
884 vm_object_offset_t first_offset, /* Offset into object */
885 vm_prot_t fault_type, /* What access is requested */
886 boolean_t must_be_resident,/* Must page be resident? */
887 boolean_t caller_lookup, /* caller looked up page */
888 /* Modifies in place: */
889 vm_prot_t *protection, /* Protection for mapping */
890 vm_page_t *result_page, /* Page found, if successful */
891 /* Returns: */
892 vm_page_t *top_page, /* Page in top object, if
893 * not result_page. */
894 int *type_of_fault, /* if non-null, fill in with type of fault
895 * COW, zero-fill, etc... returned in trace point */
896 /* More arguments: */
897 kern_return_t *error_code, /* code if page is in error */
898 boolean_t no_zero_fill, /* don't zero fill absent pages */
899 boolean_t data_supply, /* treat as data_supply if
900 * it is a write fault and a full
901 * page is provided */
902 vm_object_fault_info_t fault_info)
903 {
904 vm_page_t m;
905 vm_object_t object;
906 vm_object_offset_t offset;
907 vm_page_t first_m;
908 vm_object_t next_object;
909 vm_object_t copy_object;
910 boolean_t look_for_page;
911 boolean_t force_fault_retry = FALSE;
912 vm_prot_t access_required = fault_type;
913 vm_prot_t wants_copy_flag;
914 kern_return_t wait_result;
915 wait_interrupt_t interruptible_state;
916 boolean_t data_already_requested = FALSE;
917 vm_behavior_t orig_behavior;
918 vm_size_t orig_cluster_size;
919 vm_fault_return_t error;
920 int my_fault;
921 uint32_t try_failed_count;
922 int interruptible; /* how may fault be interrupted? */
923 int external_state = VM_EXTERNAL_STATE_UNKNOWN;
924 memory_object_t pager;
925 vm_fault_return_t retval;
926 int grab_options;
927
928 /*
929 * MUST_ASK_PAGER() evaluates to TRUE if the page specified by object/offset is
930 * marked as paged out in the compressor pager or the pager doesn't exist.
931 * Note also that if the pager for an internal object
932 * has not been created, the pager is not invoked regardless of the value
933 * of MUST_ASK_PAGER().
934 *
935 * PAGED_OUT() evaluates to TRUE if the page specified by the object/offset
936 * is marked as paged out in the compressor pager.
937 * PAGED_OUT() is used to determine if a page has already been pushed
938 * into a copy object in order to avoid a redundant page out operation.
939 */
940 #define MUST_ASK_PAGER(o, f, s) \
941 ((s = VM_COMPRESSOR_PAGER_STATE_GET((o), (f))) != VM_EXTERNAL_STATE_ABSENT)
942
943 #define PAGED_OUT(o, f) \
944 (VM_COMPRESSOR_PAGER_STATE_GET((o), (f)) == VM_EXTERNAL_STATE_EXISTS)
945
946 /*
947 * Recovery actions
948 */
949 #define RELEASE_PAGE(m) \
950 MACRO_BEGIN \
951 PAGE_WAKEUP_DONE(m); \
952 if ( !VM_PAGE_PAGEABLE(m)) { \
953 vm_page_lockspin_queues(); \
954 if ( !VM_PAGE_PAGEABLE(m)) { \
955 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) \
956 vm_page_deactivate(m); \
957 else \
958 vm_page_activate(m); \
959 } \
960 vm_page_unlock_queues(); \
961 } \
962 MACRO_END
963
964 #if TRACEFAULTPAGE
965 dbgTrace(0xBEEF0002, (unsigned int) first_object, (unsigned int) first_offset); /* (TEST/DEBUG) */
966 #endif
967
968 interruptible = fault_info->interruptible;
969 interruptible_state = thread_interrupt_level(interruptible);
970
971 /*
972 * INVARIANTS (through entire routine):
973 *
974 * 1) At all times, we must either have the object
975 * lock or a busy page in some object to prevent
976 * some other thread from trying to bring in
977 * the same page.
978 *
979 * Note that we cannot hold any locks during the
980 * pager access or when waiting for memory, so
981 * we use a busy page then.
982 *
983 * 2) To prevent another thread from racing us down the
984 * shadow chain and entering a new page in the top
985 * object before we do, we must keep a busy page in
986 * the top object while following the shadow chain.
987 *
988 * 3) We must increment paging_in_progress on any object
989 * for which we have a busy page before dropping
990 * the object lock
991 *
992 * 4) We leave busy pages on the pageout queues.
993 * If the pageout daemon comes across a busy page,
994 * it will remove the page from the pageout queues.
995 */
996
997 object = first_object;
998 offset = first_offset;
999 first_m = VM_PAGE_NULL;
1000 access_required = fault_type;
1001
1002 /*
1003 * default type of fault
1004 */
1005 my_fault = DBG_CACHE_HIT_FAULT;
1006
1007 while (TRUE) {
1008 #if TRACEFAULTPAGE
1009 dbgTrace(0xBEEF0003, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */
1010 #endif
1011
1012 grab_options = 0;
1013 #if CONFIG_SECLUDED_MEMORY
1014 if (object->can_grab_secluded) {
1015 grab_options |= VM_PAGE_GRAB_SECLUDED;
1016 }
1017 #endif /* CONFIG_SECLUDED_MEMORY */
1018
1019 if (!object->alive) {
1020 /*
1021 * object is no longer valid
1022 * clean up and return error
1023 */
1024 vm_fault_cleanup(object, first_m);
1025 thread_interrupt_level(interruptible_state);
1026
1027 return VM_FAULT_MEMORY_ERROR;
1028 }
1029
1030 if (!object->pager_created && object->phys_contiguous) {
1031 /*
1032 * A physically-contiguous object without a pager:
1033 * must be a "large page" object. We do not deal
1034 * with VM pages for this object.
1035 */
1036 caller_lookup = FALSE;
1037 m = VM_PAGE_NULL;
1038 goto phys_contig_object;
1039 }
1040
1041 if (object->blocked_access) {
1042 /*
1043 * Access to this VM object has been blocked.
1044 * Replace our "paging_in_progress" reference with
1045 * a "activity_in_progress" reference and wait for
1046 * access to be unblocked.
1047 */
1048 caller_lookup = FALSE; /* no longer valid after sleep */
1049 vm_object_activity_begin(object);
1050 vm_object_paging_end(object);
1051 while (object->blocked_access) {
1052 vm_object_sleep(object,
1053 VM_OBJECT_EVENT_UNBLOCKED,
1054 THREAD_UNINT);
1055 }
1056 vm_fault_page_blocked_access++;
1057 vm_object_paging_begin(object);
1058 vm_object_activity_end(object);
1059 }
1060
1061 /*
1062 * See whether the page at 'offset' is resident
1063 */
1064 if (caller_lookup == TRUE) {
1065 /*
1066 * The caller has already looked up the page
1067 * and gave us the result in "result_page".
1068 * We can use this for the first lookup but
1069 * it loses its validity as soon as we unlock
1070 * the object.
1071 */
1072 m = *result_page;
1073 caller_lookup = FALSE; /* no longer valid after that */
1074 } else {
1075 m = vm_page_lookup(object, offset);
1076 }
1077 #if TRACEFAULTPAGE
1078 dbgTrace(0xBEEF0004, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
1079 #endif
1080 if (m != VM_PAGE_NULL) {
1081 if (m->vmp_busy) {
1082 /*
1083 * The page is being brought in,
1084 * wait for it and then retry.
1085 */
1086 #if TRACEFAULTPAGE
1087 dbgTrace(0xBEEF0005, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1088 #endif
1089 wait_result = PAGE_SLEEP(object, m, interruptible);
1090
1091 counter(c_vm_fault_page_block_busy_kernel++);
1092
1093 if (wait_result != THREAD_AWAKENED) {
1094 vm_fault_cleanup(object, first_m);
1095 thread_interrupt_level(interruptible_state);
1096
1097 if (wait_result == THREAD_RESTART) {
1098 return VM_FAULT_RETRY;
1099 } else {
1100 return VM_FAULT_INTERRUPTED;
1101 }
1102 }
1103 continue;
1104 }
1105 if (m->vmp_laundry) {
1106 m->vmp_free_when_done = FALSE;
1107
1108 if (!m->vmp_cleaning) {
1109 vm_pageout_steal_laundry(m, FALSE);
1110 }
1111 }
1112 if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
1113 /*
1114 * Guard page: off limits !
1115 */
1116 if (fault_type == VM_PROT_NONE) {
1117 /*
1118 * The fault is not requesting any
1119 * access to the guard page, so it must
1120 * be just to wire or unwire it.
1121 * Let's pretend it succeeded...
1122 */
1123 m->vmp_busy = TRUE;
1124 *result_page = m;
1125 assert(first_m == VM_PAGE_NULL);
1126 *top_page = first_m;
1127 if (type_of_fault) {
1128 *type_of_fault = DBG_GUARD_FAULT;
1129 }
1130 thread_interrupt_level(interruptible_state);
1131 return VM_FAULT_SUCCESS;
1132 } else {
1133 /*
1134 * The fault requests access to the
1135 * guard page: let's deny that !
1136 */
1137 vm_fault_cleanup(object, first_m);
1138 thread_interrupt_level(interruptible_state);
1139 return VM_FAULT_MEMORY_ERROR;
1140 }
1141 }
1142
1143 if (m->vmp_error) {
1144 /*
1145 * The page is in error, give up now.
1146 */
1147 #if TRACEFAULTPAGE
1148 dbgTrace(0xBEEF0006, (unsigned int) m, (unsigned int) error_code); /* (TEST/DEBUG) */
1149 #endif
1150 if (error_code) {
1151 *error_code = KERN_MEMORY_ERROR;
1152 }
1153 VM_PAGE_FREE(m);
1154
1155 vm_fault_cleanup(object, first_m);
1156 thread_interrupt_level(interruptible_state);
1157
1158 return VM_FAULT_MEMORY_ERROR;
1159 }
1160 if (m->vmp_restart) {
1161 /*
1162 * The pager wants us to restart
1163 * at the top of the chain,
1164 * typically because it has moved the
1165 * page to another pager, then do so.
1166 */
1167 #if TRACEFAULTPAGE
1168 dbgTrace(0xBEEF0007, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1169 #endif
1170 VM_PAGE_FREE(m);
1171
1172 vm_fault_cleanup(object, first_m);
1173 thread_interrupt_level(interruptible_state);
1174
1175 return VM_FAULT_RETRY;
1176 }
1177 if (m->vmp_absent) {
1178 /*
1179 * The page isn't busy, but is absent,
1180 * therefore it's deemed "unavailable".
1181 *
1182 * Remove the non-existent page (unless it's
1183 * in the top object) and move on down to the
1184 * next object (if there is one).
1185 */
1186 #if TRACEFAULTPAGE
1187 dbgTrace(0xBEEF0008, (unsigned int) m, (unsigned int) object->shadow); /* (TEST/DEBUG) */
1188 #endif
1189 next_object = object->shadow;
1190
1191 if (next_object == VM_OBJECT_NULL) {
1192 /*
1193 * Absent page at bottom of shadow
1194 * chain; zero fill the page we left
1195 * busy in the first object, and free
1196 * the absent page.
1197 */
1198 assert(!must_be_resident);
1199
1200 /*
1201 * check for any conditions that prevent
1202 * us from creating a new zero-fill page
1203 * vm_fault_check will do all of the
1204 * fault cleanup in the case of an error condition
1205 * including resetting the thread_interrupt_level
1206 */
1207 error = vm_fault_check(object, m, first_m, interruptible_state, (type_of_fault == NULL) ? TRUE : FALSE);
1208
1209 if (error != VM_FAULT_SUCCESS) {
1210 return error;
1211 }
1212
1213 if (object != first_object) {
1214 /*
1215 * free the absent page we just found
1216 */
1217 VM_PAGE_FREE(m);
1218
1219 /*
1220 * drop reference and lock on current object
1221 */
1222 vm_object_paging_end(object);
1223 vm_object_unlock(object);
1224
1225 /*
1226 * grab the original page we
1227 * 'soldered' in place and
1228 * retake lock on 'first_object'
1229 */
1230 m = first_m;
1231 first_m = VM_PAGE_NULL;
1232
1233 object = first_object;
1234 offset = first_offset;
1235
1236 vm_object_lock(object);
1237 } else {
1238 /*
1239 * we're going to use the absent page we just found
1240 * so convert it to a 'busy' page
1241 */
1242 m->vmp_absent = FALSE;
1243 m->vmp_busy = TRUE;
1244 }
1245 if (fault_info->mark_zf_absent && no_zero_fill == TRUE) {
1246 m->vmp_absent = TRUE;
1247 }
1248 /*
1249 * zero-fill the page and put it on
1250 * the correct paging queue
1251 */
1252 my_fault = vm_fault_zero_page(m, no_zero_fill);
1253
1254 break;
1255 } else {
1256 if (must_be_resident) {
1257 vm_object_paging_end(object);
1258 } else if (object != first_object) {
1259 vm_object_paging_end(object);
1260 VM_PAGE_FREE(m);
1261 } else {
1262 first_m = m;
1263 m->vmp_absent = FALSE;
1264 m->vmp_busy = TRUE;
1265
1266 vm_page_lockspin_queues();
1267 vm_page_queues_remove(m, FALSE);
1268 vm_page_unlock_queues();
1269 }
1270
1271 offset += object->vo_shadow_offset;
1272 fault_info->lo_offset += object->vo_shadow_offset;
1273 fault_info->hi_offset += object->vo_shadow_offset;
1274 access_required = VM_PROT_READ;
1275
1276 vm_object_lock(next_object);
1277 vm_object_unlock(object);
1278 object = next_object;
1279 vm_object_paging_begin(object);
1280
1281 /*
1282 * reset to default type of fault
1283 */
1284 my_fault = DBG_CACHE_HIT_FAULT;
1285
1286 continue;
1287 }
1288 }
1289 if ((m->vmp_cleaning)
1290 && ((object != first_object) || (object->copy != VM_OBJECT_NULL))
1291 && (fault_type & VM_PROT_WRITE)) {
1292 /*
1293 * This is a copy-on-write fault that will
1294 * cause us to revoke access to this page, but
1295 * this page is in the process of being cleaned
1296 * in a clustered pageout. We must wait until
1297 * the cleaning operation completes before
1298 * revoking access to the original page,
1299 * otherwise we might attempt to remove a
1300 * wired mapping.
1301 */
1302 #if TRACEFAULTPAGE
1303 dbgTrace(0xBEEF0009, (unsigned int) m, (unsigned int) offset); /* (TEST/DEBUG) */
1304 #endif
1305 /*
1306 * take an extra ref so that object won't die
1307 */
1308 vm_object_reference_locked(object);
1309
1310 vm_fault_cleanup(object, first_m);
1311
1312 counter(c_vm_fault_page_block_backoff_kernel++);
1313 vm_object_lock(object);
1314 assert(object->ref_count > 0);
1315
1316 m = vm_page_lookup(object, offset);
1317
1318 if (m != VM_PAGE_NULL && m->vmp_cleaning) {
1319 PAGE_ASSERT_WAIT(m, interruptible);
1320
1321 vm_object_unlock(object);
1322 wait_result = thread_block(THREAD_CONTINUE_NULL);
1323 vm_object_deallocate(object);
1324
1325 goto backoff;
1326 } else {
1327 vm_object_unlock(object);
1328
1329 vm_object_deallocate(object);
1330 thread_interrupt_level(interruptible_state);
1331
1332 return VM_FAULT_RETRY;
1333 }
1334 }
1335 if (type_of_fault == NULL && (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) &&
1336 !(fault_info != NULL && fault_info->stealth)) {
1337 /*
1338 * If we were passed a non-NULL pointer for
1339 * "type_of_fault", than we came from
1340 * vm_fault... we'll let it deal with
1341 * this condition, since it
1342 * needs to see m->vmp_speculative to correctly
1343 * account the pageins, otherwise...
1344 * take it off the speculative queue, we'll
1345 * let the caller of vm_fault_page deal
1346 * with getting it onto the correct queue
1347 *
1348 * If the caller specified in fault_info that
1349 * it wants a "stealth" fault, we also leave
1350 * the page in the speculative queue.
1351 */
1352 vm_page_lockspin_queues();
1353 if (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
1354 vm_page_queues_remove(m, FALSE);
1355 }
1356 vm_page_unlock_queues();
1357 }
1358 assert(object == VM_PAGE_OBJECT(m));
1359
1360 if (object->code_signed) {
1361 /*
1362 * CODE SIGNING:
1363 * We just paged in a page from a signed
1364 * memory object but we don't need to
1365 * validate it now. We'll validate it if
1366 * when it gets mapped into a user address
1367 * space for the first time or when the page
1368 * gets copied to another object as a result
1369 * of a copy-on-write.
1370 */
1371 }
1372
1373 /*
1374 * We mark the page busy and leave it on
1375 * the pageout queues. If the pageout
1376 * deamon comes across it, then it will
1377 * remove the page from the queue, but not the object
1378 */
1379 #if TRACEFAULTPAGE
1380 dbgTrace(0xBEEF000B, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1381 #endif
1382 assert(!m->vmp_busy);
1383 assert(!m->vmp_absent);
1384
1385 m->vmp_busy = TRUE;
1386 break;
1387 }
1388
1389
1390 /*
1391 * we get here when there is no page present in the object at
1392 * the offset we're interested in... we'll allocate a page
1393 * at this point if the pager associated with
1394 * this object can provide the data or we're the top object...
1395 * object is locked; m == NULL
1396 */
1397
1398 if (must_be_resident) {
1399 if (fault_type == VM_PROT_NONE &&
1400 object == kernel_object) {
1401 /*
1402 * We've been called from vm_fault_unwire()
1403 * while removing a map entry that was allocated
1404 * with KMA_KOBJECT and KMA_VAONLY. This page
1405 * is not present and there's nothing more to
1406 * do here (nothing to unwire).
1407 */
1408 vm_fault_cleanup(object, first_m);
1409 thread_interrupt_level(interruptible_state);
1410
1411 return VM_FAULT_MEMORY_ERROR;
1412 }
1413
1414 goto dont_look_for_page;
1415 }
1416
1417 /* Don't expect to fault pages into the kernel object. */
1418 assert(object != kernel_object);
1419
1420 data_supply = FALSE;
1421
1422 look_for_page = (object->pager_created && (MUST_ASK_PAGER(object, offset, external_state) == TRUE) && !data_supply);
1423
1424 #if TRACEFAULTPAGE
1425 dbgTrace(0xBEEF000C, (unsigned int) look_for_page, (unsigned int) object); /* (TEST/DEBUG) */
1426 #endif
1427 if (!look_for_page && object == first_object && !object->phys_contiguous) {
1428 /*
1429 * Allocate a new page for this object/offset pair as a placeholder
1430 */
1431 m = vm_page_grab_options(grab_options);
1432 #if TRACEFAULTPAGE
1433 dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
1434 #endif
1435 if (m == VM_PAGE_NULL) {
1436 vm_fault_cleanup(object, first_m);
1437 thread_interrupt_level(interruptible_state);
1438
1439 return VM_FAULT_MEMORY_SHORTAGE;
1440 }
1441
1442 if (fault_info && fault_info->batch_pmap_op == TRUE) {
1443 vm_page_insert_internal(m, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, TRUE, FALSE, NULL);
1444 } else {
1445 vm_page_insert(m, object, offset);
1446 }
1447 }
1448 if (look_for_page) {
1449 kern_return_t rc;
1450 int my_fault_type;
1451
1452 /*
1453 * If the memory manager is not ready, we
1454 * cannot make requests.
1455 */
1456 if (!object->pager_ready) {
1457 #if TRACEFAULTPAGE
1458 dbgTrace(0xBEEF000E, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */
1459 #endif
1460 if (m != VM_PAGE_NULL) {
1461 VM_PAGE_FREE(m);
1462 }
1463
1464 /*
1465 * take an extra ref so object won't die
1466 */
1467 vm_object_reference_locked(object);
1468 vm_fault_cleanup(object, first_m);
1469 counter(c_vm_fault_page_block_backoff_kernel++);
1470
1471 vm_object_lock(object);
1472 assert(object->ref_count > 0);
1473
1474 if (!object->pager_ready) {
1475 wait_result = vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGER_READY, interruptible);
1476
1477 vm_object_unlock(object);
1478 if (wait_result == THREAD_WAITING) {
1479 wait_result = thread_block(THREAD_CONTINUE_NULL);
1480 }
1481 vm_object_deallocate(object);
1482
1483 goto backoff;
1484 } else {
1485 vm_object_unlock(object);
1486 vm_object_deallocate(object);
1487 thread_interrupt_level(interruptible_state);
1488
1489 return VM_FAULT_RETRY;
1490 }
1491 }
1492 if (!object->internal && !object->phys_contiguous && object->paging_in_progress > vm_object_pagein_throttle) {
1493 /*
1494 * If there are too many outstanding page
1495 * requests pending on this external object, we
1496 * wait for them to be resolved now.
1497 */
1498 #if TRACEFAULTPAGE
1499 dbgTrace(0xBEEF0010, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1500 #endif
1501 if (m != VM_PAGE_NULL) {
1502 VM_PAGE_FREE(m);
1503 }
1504 /*
1505 * take an extra ref so object won't die
1506 */
1507 vm_object_reference_locked(object);
1508
1509 vm_fault_cleanup(object, first_m);
1510
1511 counter(c_vm_fault_page_block_backoff_kernel++);
1512
1513 vm_object_lock(object);
1514 assert(object->ref_count > 0);
1515
1516 if (object->paging_in_progress >= vm_object_pagein_throttle) {
1517 vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS, interruptible);
1518
1519 vm_object_unlock(object);
1520 wait_result = thread_block(THREAD_CONTINUE_NULL);
1521 vm_object_deallocate(object);
1522
1523 goto backoff;
1524 } else {
1525 vm_object_unlock(object);
1526 vm_object_deallocate(object);
1527 thread_interrupt_level(interruptible_state);
1528
1529 return VM_FAULT_RETRY;
1530 }
1531 }
1532 if (object->internal) {
1533 int compressed_count_delta;
1534
1535 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
1536
1537 if (m == VM_PAGE_NULL) {
1538 /*
1539 * Allocate a new page for this object/offset pair as a placeholder
1540 */
1541 m = vm_page_grab_options(grab_options);
1542 #if TRACEFAULTPAGE
1543 dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
1544 #endif
1545 if (m == VM_PAGE_NULL) {
1546 vm_fault_cleanup(object, first_m);
1547 thread_interrupt_level(interruptible_state);
1548
1549 return VM_FAULT_MEMORY_SHORTAGE;
1550 }
1551
1552 m->vmp_absent = TRUE;
1553 if (fault_info && fault_info->batch_pmap_op == TRUE) {
1554 vm_page_insert_internal(m, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, TRUE, FALSE, NULL);
1555 } else {
1556 vm_page_insert(m, object, offset);
1557 }
1558 }
1559 assert(m->vmp_busy);
1560
1561 m->vmp_absent = TRUE;
1562 pager = object->pager;
1563
1564 assert(object->paging_in_progress > 0);
1565 vm_object_unlock(object);
1566
1567 rc = vm_compressor_pager_get(
1568 pager,
1569 offset + object->paging_offset,
1570 VM_PAGE_GET_PHYS_PAGE(m),
1571 &my_fault_type,
1572 0,
1573 &compressed_count_delta);
1574
1575 if (type_of_fault == NULL) {
1576 int throttle_delay;
1577
1578 /*
1579 * we weren't called from vm_fault, so we
1580 * need to apply page creation throttling
1581 * do it before we re-acquire any locks
1582 */
1583 if (my_fault_type == DBG_COMPRESSOR_FAULT) {
1584 if ((throttle_delay = vm_page_throttled(TRUE))) {
1585 VM_DEBUG_EVENT(vmf_compressordelay, VMF_COMPRESSORDELAY, DBG_FUNC_NONE, throttle_delay, 0, 1, 0);
1586 delay(throttle_delay);
1587 }
1588 }
1589 }
1590 vm_object_lock(object);
1591 assert(object->paging_in_progress > 0);
1592
1593 vm_compressor_pager_count(
1594 pager,
1595 compressed_count_delta,
1596 FALSE, /* shared_lock */
1597 object);
1598
1599 switch (rc) {
1600 case KERN_SUCCESS:
1601 m->vmp_absent = FALSE;
1602 m->vmp_dirty = TRUE;
1603 if ((object->wimg_bits &
1604 VM_WIMG_MASK) !=
1605 VM_WIMG_USE_DEFAULT) {
1606 /*
1607 * If the page is not cacheable,
1608 * we can't let its contents
1609 * linger in the data cache
1610 * after the decompression.
1611 */
1612 pmap_sync_page_attributes_phys(
1613 VM_PAGE_GET_PHYS_PAGE(m));
1614 } else {
1615 m->vmp_written_by_kernel = TRUE;
1616 }
1617
1618 /*
1619 * If the object is purgeable, its
1620 * owner's purgeable ledgers have been
1621 * updated in vm_page_insert() but the
1622 * page was also accounted for in a
1623 * "compressed purgeable" ledger, so
1624 * update that now.
1625 */
1626 if (((object->purgable !=
1627 VM_PURGABLE_DENY) ||
1628 object->vo_ledger_tag) &&
1629 (object->vo_owner !=
1630 NULL)) {
1631 /*
1632 * One less compressed
1633 * purgeable/tagged page.
1634 */
1635 vm_object_owner_compressed_update(
1636 object,
1637 -1);
1638 }
1639
1640 break;
1641 case KERN_MEMORY_FAILURE:
1642 m->vmp_unusual = TRUE;
1643 m->vmp_error = TRUE;
1644 m->vmp_absent = FALSE;
1645 break;
1646 case KERN_MEMORY_ERROR:
1647 assert(m->vmp_absent);
1648 break;
1649 default:
1650 panic("vm_fault_page(): unexpected "
1651 "error %d from "
1652 "vm_compressor_pager_get()\n",
1653 rc);
1654 }
1655 PAGE_WAKEUP_DONE(m);
1656
1657 rc = KERN_SUCCESS;
1658 goto data_requested;
1659 }
1660 my_fault_type = DBG_PAGEIN_FAULT;
1661
1662 if (m != VM_PAGE_NULL) {
1663 VM_PAGE_FREE(m);
1664 m = VM_PAGE_NULL;
1665 }
1666
1667 #if TRACEFAULTPAGE
1668 dbgTrace(0xBEEF0012, (unsigned int) object, (unsigned int) 0); /* (TEST/DEBUG) */
1669 #endif
1670
1671 /*
1672 * It's possible someone called vm_object_destroy while we weren't
1673 * holding the object lock. If that has happened, then bail out
1674 * here.
1675 */
1676
1677 pager = object->pager;
1678
1679 if (pager == MEMORY_OBJECT_NULL) {
1680 vm_fault_cleanup(object, first_m);
1681 thread_interrupt_level(interruptible_state);
1682 return VM_FAULT_MEMORY_ERROR;
1683 }
1684
1685 /*
1686 * We have an absent page in place for the faulting offset,
1687 * so we can release the object lock.
1688 */
1689
1690 if (object->object_is_shared_cache) {
1691 set_thread_rwlock_boost();
1692 }
1693
1694 vm_object_unlock(object);
1695
1696 /*
1697 * If this object uses a copy_call strategy,
1698 * and we are interested in a copy of this object
1699 * (having gotten here only by following a
1700 * shadow chain), then tell the memory manager
1701 * via a flag added to the desired_access
1702 * parameter, so that it can detect a race
1703 * between our walking down the shadow chain
1704 * and its pushing pages up into a copy of
1705 * the object that it manages.
1706 */
1707 if (object->copy_strategy == MEMORY_OBJECT_COPY_CALL && object != first_object) {
1708 wants_copy_flag = VM_PROT_WANTS_COPY;
1709 } else {
1710 wants_copy_flag = VM_PROT_NONE;
1711 }
1712
1713 if (object->copy == first_object) {
1714 /*
1715 * if we issue the memory_object_data_request in
1716 * this state, we are subject to a deadlock with
1717 * the underlying filesystem if it is trying to
1718 * shrink the file resulting in a push of pages
1719 * into the copy object... that push will stall
1720 * on the placeholder page, and if the pushing thread
1721 * is holding a lock that is required on the pagein
1722 * path (such as a truncate lock), we'll deadlock...
1723 * to avoid this potential deadlock, we throw away
1724 * our placeholder page before calling memory_object_data_request
1725 * and force this thread to retry the vm_fault_page after
1726 * we have issued the I/O. the second time through this path
1727 * we will find the page already in the cache (presumably still
1728 * busy waiting for the I/O to complete) and then complete
1729 * the fault w/o having to go through memory_object_data_request again
1730 */
1731 assert(first_m != VM_PAGE_NULL);
1732 assert(VM_PAGE_OBJECT(first_m) == first_object);
1733
1734 vm_object_lock(first_object);
1735 VM_PAGE_FREE(first_m);
1736 vm_object_paging_end(first_object);
1737 vm_object_unlock(first_object);
1738
1739 first_m = VM_PAGE_NULL;
1740 force_fault_retry = TRUE;
1741
1742 vm_fault_page_forced_retry++;
1743 }
1744
1745 if (data_already_requested == TRUE) {
1746 orig_behavior = fault_info->behavior;
1747 orig_cluster_size = fault_info->cluster_size;
1748
1749 fault_info->behavior = VM_BEHAVIOR_RANDOM;
1750 fault_info->cluster_size = PAGE_SIZE;
1751 }
1752 /*
1753 * Call the memory manager to retrieve the data.
1754 */
1755 rc = memory_object_data_request(
1756 pager,
1757 offset + object->paging_offset,
1758 PAGE_SIZE,
1759 access_required | wants_copy_flag,
1760 (memory_object_fault_info_t)fault_info);
1761
1762 if (data_already_requested == TRUE) {
1763 fault_info->behavior = orig_behavior;
1764 fault_info->cluster_size = orig_cluster_size;
1765 } else {
1766 data_already_requested = TRUE;
1767 }
1768
1769 DTRACE_VM2(maj_fault, int, 1, (uint64_t *), NULL);
1770 #if TRACEFAULTPAGE
1771 dbgTrace(0xBEEF0013, (unsigned int) object, (unsigned int) rc); /* (TEST/DEBUG) */
1772 #endif
1773 vm_object_lock(object);
1774
1775 if (object->object_is_shared_cache) {
1776 clear_thread_rwlock_boost();
1777 }
1778
1779 data_requested:
1780 if (rc != KERN_SUCCESS) {
1781 vm_fault_cleanup(object, first_m);
1782 thread_interrupt_level(interruptible_state);
1783
1784 return (rc == MACH_SEND_INTERRUPTED) ?
1785 VM_FAULT_INTERRUPTED :
1786 VM_FAULT_MEMORY_ERROR;
1787 } else {
1788 clock_sec_t tv_sec;
1789 clock_usec_t tv_usec;
1790
1791 if (my_fault_type == DBG_PAGEIN_FAULT) {
1792 clock_get_system_microtime(&tv_sec, &tv_usec);
1793 current_thread()->t_page_creation_time = tv_sec;
1794 current_thread()->t_page_creation_count = 0;
1795 }
1796 }
1797 if ((interruptible != THREAD_UNINT) && (current_thread()->sched_flags & TH_SFLAG_ABORT)) {
1798 vm_fault_cleanup(object, first_m);
1799 thread_interrupt_level(interruptible_state);
1800
1801 return VM_FAULT_INTERRUPTED;
1802 }
1803 if (force_fault_retry == TRUE) {
1804 vm_fault_cleanup(object, first_m);
1805 thread_interrupt_level(interruptible_state);
1806
1807 return VM_FAULT_RETRY;
1808 }
1809 if (m == VM_PAGE_NULL && object->phys_contiguous) {
1810 /*
1811 * No page here means that the object we
1812 * initially looked up was "physically
1813 * contiguous" (i.e. device memory). However,
1814 * with Virtual VRAM, the object might not
1815 * be backed by that device memory anymore,
1816 * so we're done here only if the object is
1817 * still "phys_contiguous".
1818 * Otherwise, if the object is no longer
1819 * "phys_contiguous", we need to retry the
1820 * page fault against the object's new backing
1821 * store (different memory object).
1822 */
1823 phys_contig_object:
1824 goto done;
1825 }
1826 /*
1827 * potentially a pagein fault
1828 * if we make it through the state checks
1829 * above, than we'll count it as such
1830 */
1831 my_fault = my_fault_type;
1832
1833 /*
1834 * Retry with same object/offset, since new data may
1835 * be in a different page (i.e., m is meaningless at
1836 * this point).
1837 */
1838 continue;
1839 }
1840 dont_look_for_page:
1841 /*
1842 * We get here if the object has no pager, or an existence map
1843 * exists and indicates the page isn't present on the pager
1844 * or we're unwiring a page. If a pager exists, but there
1845 * is no existence map, then the m->vmp_absent case above handles
1846 * the ZF case when the pager can't provide the page
1847 */
1848 #if TRACEFAULTPAGE
1849 dbgTrace(0xBEEF0014, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
1850 #endif
1851 if (object == first_object) {
1852 first_m = m;
1853 } else {
1854 assert(m == VM_PAGE_NULL);
1855 }
1856
1857 next_object = object->shadow;
1858
1859 if (next_object == VM_OBJECT_NULL) {
1860 /*
1861 * we've hit the bottom of the shadown chain,
1862 * fill the page in the top object with zeros.
1863 */
1864 assert(!must_be_resident);
1865
1866 if (object != first_object) {
1867 vm_object_paging_end(object);
1868 vm_object_unlock(object);
1869
1870 object = first_object;
1871 offset = first_offset;
1872 vm_object_lock(object);
1873 }
1874 m = first_m;
1875 assert(VM_PAGE_OBJECT(m) == object);
1876 first_m = VM_PAGE_NULL;
1877
1878 /*
1879 * check for any conditions that prevent
1880 * us from creating a new zero-fill page
1881 * vm_fault_check will do all of the
1882 * fault cleanup in the case of an error condition
1883 * including resetting the thread_interrupt_level
1884 */
1885 error = vm_fault_check(object, m, first_m, interruptible_state, (type_of_fault == NULL) ? TRUE : FALSE);
1886
1887 if (error != VM_FAULT_SUCCESS) {
1888 return error;
1889 }
1890
1891 if (m == VM_PAGE_NULL) {
1892 m = vm_page_grab_options(grab_options);
1893
1894 if (m == VM_PAGE_NULL) {
1895 vm_fault_cleanup(object, VM_PAGE_NULL);
1896 thread_interrupt_level(interruptible_state);
1897
1898 return VM_FAULT_MEMORY_SHORTAGE;
1899 }
1900 vm_page_insert(m, object, offset);
1901 }
1902 if (fault_info->mark_zf_absent && no_zero_fill == TRUE) {
1903 m->vmp_absent = TRUE;
1904 }
1905
1906 my_fault = vm_fault_zero_page(m, no_zero_fill);
1907
1908 break;
1909 } else {
1910 /*
1911 * Move on to the next object. Lock the next
1912 * object before unlocking the current one.
1913 */
1914 if ((object != first_object) || must_be_resident) {
1915 vm_object_paging_end(object);
1916 }
1917
1918 offset += object->vo_shadow_offset;
1919 fault_info->lo_offset += object->vo_shadow_offset;
1920 fault_info->hi_offset += object->vo_shadow_offset;
1921 access_required = VM_PROT_READ;
1922
1923 vm_object_lock(next_object);
1924 vm_object_unlock(object);
1925
1926 object = next_object;
1927 vm_object_paging_begin(object);
1928 }
1929 }
1930
1931 /*
1932 * PAGE HAS BEEN FOUND.
1933 *
1934 * This page (m) is:
1935 * busy, so that we can play with it;
1936 * not absent, so that nobody else will fill it;
1937 * possibly eligible for pageout;
1938 *
1939 * The top-level page (first_m) is:
1940 * VM_PAGE_NULL if the page was found in the
1941 * top-level object;
1942 * busy, not absent, and ineligible for pageout.
1943 *
1944 * The current object (object) is locked. A paging
1945 * reference is held for the current and top-level
1946 * objects.
1947 */
1948
1949 #if TRACEFAULTPAGE
1950 dbgTrace(0xBEEF0015, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
1951 #endif
1952 #if EXTRA_ASSERTIONS
1953 assert(m->vmp_busy && !m->vmp_absent);
1954 assert((first_m == VM_PAGE_NULL) ||
1955 (first_m->vmp_busy && !first_m->vmp_absent &&
1956 !first_m->vmp_active && !first_m->vmp_inactive && !first_m->vmp_secluded));
1957 #endif /* EXTRA_ASSERTIONS */
1958
1959 /*
1960 * If the page is being written, but isn't
1961 * already owned by the top-level object,
1962 * we have to copy it into a new page owned
1963 * by the top-level object.
1964 */
1965 if (object != first_object) {
1966 #if TRACEFAULTPAGE
1967 dbgTrace(0xBEEF0016, (unsigned int) object, (unsigned int) fault_type); /* (TEST/DEBUG) */
1968 #endif
1969 if (fault_type & VM_PROT_WRITE) {
1970 vm_page_t copy_m;
1971
1972 /*
1973 * We only really need to copy if we
1974 * want to write it.
1975 */
1976 assert(!must_be_resident);
1977
1978 /*
1979 * If we try to collapse first_object at this
1980 * point, we may deadlock when we try to get
1981 * the lock on an intermediate object (since we
1982 * have the bottom object locked). We can't
1983 * unlock the bottom object, because the page
1984 * we found may move (by collapse) if we do.
1985 *
1986 * Instead, we first copy the page. Then, when
1987 * we have no more use for the bottom object,
1988 * we unlock it and try to collapse.
1989 *
1990 * Note that we copy the page even if we didn't
1991 * need to... that's the breaks.
1992 */
1993
1994 /*
1995 * Allocate a page for the copy
1996 */
1997 copy_m = vm_page_grab_options(grab_options);
1998
1999 if (copy_m == VM_PAGE_NULL) {
2000 RELEASE_PAGE(m);
2001
2002 vm_fault_cleanup(object, first_m);
2003 thread_interrupt_level(interruptible_state);
2004
2005 return VM_FAULT_MEMORY_SHORTAGE;
2006 }
2007
2008 vm_page_copy(m, copy_m);
2009
2010 /*
2011 * If another map is truly sharing this
2012 * page with us, we have to flush all
2013 * uses of the original page, since we
2014 * can't distinguish those which want the
2015 * original from those which need the
2016 * new copy.
2017 *
2018 * XXXO If we know that only one map has
2019 * access to this page, then we could
2020 * avoid the pmap_disconnect() call.
2021 */
2022 if (m->vmp_pmapped) {
2023 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
2024 }
2025
2026 if (m->vmp_clustered) {
2027 VM_PAGE_COUNT_AS_PAGEIN(m);
2028 VM_PAGE_CONSUME_CLUSTERED(m);
2029 }
2030 assert(!m->vmp_cleaning);
2031
2032 /*
2033 * We no longer need the old page or object.
2034 */
2035 RELEASE_PAGE(m);
2036
2037 /*
2038 * This check helps with marking the object as having a sequential pattern
2039 * Normally we'll miss doing this below because this fault is about COW to
2040 * the first_object i.e. bring page in from disk, push to object above but
2041 * don't update the file object's sequential pattern.
2042 */
2043 if (object->internal == FALSE) {
2044 vm_fault_is_sequential(object, offset, fault_info->behavior);
2045 }
2046
2047 vm_object_paging_end(object);
2048 vm_object_unlock(object);
2049
2050 my_fault = DBG_COW_FAULT;
2051 VM_STAT_INCR(cow_faults);
2052 DTRACE_VM2(cow_fault, int, 1, (uint64_t *), NULL);
2053 current_task()->cow_faults++;
2054
2055 object = first_object;
2056 offset = first_offset;
2057
2058 vm_object_lock(object);
2059 /*
2060 * get rid of the place holder
2061 * page that we soldered in earlier
2062 */
2063 VM_PAGE_FREE(first_m);
2064 first_m = VM_PAGE_NULL;
2065
2066 /*
2067 * and replace it with the
2068 * page we just copied into
2069 */
2070 assert(copy_m->vmp_busy);
2071 vm_page_insert(copy_m, object, offset);
2072 SET_PAGE_DIRTY(copy_m, TRUE);
2073
2074 m = copy_m;
2075 /*
2076 * Now that we've gotten the copy out of the
2077 * way, let's try to collapse the top object.
2078 * But we have to play ugly games with
2079 * paging_in_progress to do that...
2080 */
2081 vm_object_paging_end(object);
2082 vm_object_collapse(object, offset, TRUE);
2083 vm_object_paging_begin(object);
2084 } else {
2085 *protection &= (~VM_PROT_WRITE);
2086 }
2087 }
2088 /*
2089 * Now check whether the page needs to be pushed into the
2090 * copy object. The use of asymmetric copy on write for
2091 * shared temporary objects means that we may do two copies to
2092 * satisfy the fault; one above to get the page from a
2093 * shadowed object, and one here to push it into the copy.
2094 */
2095 try_failed_count = 0;
2096
2097 while ((copy_object = first_object->copy) != VM_OBJECT_NULL) {
2098 vm_object_offset_t copy_offset;
2099 vm_page_t copy_m;
2100
2101 #if TRACEFAULTPAGE
2102 dbgTrace(0xBEEF0017, (unsigned int) copy_object, (unsigned int) fault_type); /* (TEST/DEBUG) */
2103 #endif
2104 /*
2105 * If the page is being written, but hasn't been
2106 * copied to the copy-object, we have to copy it there.
2107 */
2108 if ((fault_type & VM_PROT_WRITE) == 0) {
2109 *protection &= ~VM_PROT_WRITE;
2110 break;
2111 }
2112
2113 /*
2114 * If the page was guaranteed to be resident,
2115 * we must have already performed the copy.
2116 */
2117 if (must_be_resident) {
2118 break;
2119 }
2120
2121 /*
2122 * Try to get the lock on the copy_object.
2123 */
2124 if (!vm_object_lock_try(copy_object)) {
2125 vm_object_unlock(object);
2126 try_failed_count++;
2127
2128 mutex_pause(try_failed_count); /* wait a bit */
2129 vm_object_lock(object);
2130
2131 continue;
2132 }
2133 try_failed_count = 0;
2134
2135 /*
2136 * Make another reference to the copy-object,
2137 * to keep it from disappearing during the
2138 * copy.
2139 */
2140 vm_object_reference_locked(copy_object);
2141
2142 /*
2143 * Does the page exist in the copy?
2144 */
2145 copy_offset = first_offset - copy_object->vo_shadow_offset;
2146
2147 if (copy_object->vo_size <= copy_offset) {
2148 /*
2149 * Copy object doesn't cover this page -- do nothing.
2150 */
2151 ;
2152 } else if ((copy_m = vm_page_lookup(copy_object, copy_offset)) != VM_PAGE_NULL) {
2153 /*
2154 * Page currently exists in the copy object
2155 */
2156 if (copy_m->vmp_busy) {
2157 /*
2158 * If the page is being brought
2159 * in, wait for it and then retry.
2160 */
2161 RELEASE_PAGE(m);
2162
2163 /*
2164 * take an extra ref so object won't die
2165 */
2166 vm_object_reference_locked(copy_object);
2167 vm_object_unlock(copy_object);
2168 vm_fault_cleanup(object, first_m);
2169 counter(c_vm_fault_page_block_backoff_kernel++);
2170
2171 vm_object_lock(copy_object);
2172 assert(copy_object->ref_count > 0);
2173 VM_OBJ_RES_DECR(copy_object);
2174 vm_object_lock_assert_exclusive(copy_object);
2175 copy_object->ref_count--;
2176 assert(copy_object->ref_count > 0);
2177 copy_m = vm_page_lookup(copy_object, copy_offset);
2178
2179 if (copy_m != VM_PAGE_NULL && copy_m->vmp_busy) {
2180 PAGE_ASSERT_WAIT(copy_m, interruptible);
2181
2182 vm_object_unlock(copy_object);
2183 wait_result = thread_block(THREAD_CONTINUE_NULL);
2184 vm_object_deallocate(copy_object);
2185
2186 goto backoff;
2187 } else {
2188 vm_object_unlock(copy_object);
2189 vm_object_deallocate(copy_object);
2190 thread_interrupt_level(interruptible_state);
2191
2192 return VM_FAULT_RETRY;
2193 }
2194 }
2195 } else if (!PAGED_OUT(copy_object, copy_offset)) {
2196 /*
2197 * If PAGED_OUT is TRUE, then the page used to exist
2198 * in the copy-object, and has already been paged out.
2199 * We don't need to repeat this. If PAGED_OUT is
2200 * FALSE, then either we don't know (!pager_created,
2201 * for example) or it hasn't been paged out.
2202 * (VM_EXTERNAL_STATE_UNKNOWN||VM_EXTERNAL_STATE_ABSENT)
2203 * We must copy the page to the copy object.
2204 *
2205 * Allocate a page for the copy
2206 */
2207 copy_m = vm_page_alloc(copy_object, copy_offset);
2208
2209 if (copy_m == VM_PAGE_NULL) {
2210 RELEASE_PAGE(m);
2211
2212 VM_OBJ_RES_DECR(copy_object);
2213 vm_object_lock_assert_exclusive(copy_object);
2214 copy_object->ref_count--;
2215 assert(copy_object->ref_count > 0);
2216
2217 vm_object_unlock(copy_object);
2218 vm_fault_cleanup(object, first_m);
2219 thread_interrupt_level(interruptible_state);
2220
2221 return VM_FAULT_MEMORY_SHORTAGE;
2222 }
2223 /*
2224 * Must copy page into copy-object.
2225 */
2226 vm_page_copy(m, copy_m);
2227
2228 /*
2229 * If the old page was in use by any users
2230 * of the copy-object, it must be removed
2231 * from all pmaps. (We can't know which
2232 * pmaps use it.)
2233 */
2234 if (m->vmp_pmapped) {
2235 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
2236 }
2237
2238 if (m->vmp_clustered) {
2239 VM_PAGE_COUNT_AS_PAGEIN(m);
2240 VM_PAGE_CONSUME_CLUSTERED(m);
2241 }
2242 /*
2243 * If there's a pager, then immediately
2244 * page out this page, using the "initialize"
2245 * option. Else, we use the copy.
2246 */
2247 if ((!copy_object->pager_ready)
2248 || VM_COMPRESSOR_PAGER_STATE_GET(copy_object, copy_offset) == VM_EXTERNAL_STATE_ABSENT
2249 ) {
2250 vm_page_lockspin_queues();
2251 assert(!m->vmp_cleaning);
2252 vm_page_activate(copy_m);
2253 vm_page_unlock_queues();
2254
2255 SET_PAGE_DIRTY(copy_m, TRUE);
2256 PAGE_WAKEUP_DONE(copy_m);
2257 } else {
2258 assert(copy_m->vmp_busy == TRUE);
2259 assert(!m->vmp_cleaning);
2260
2261 /*
2262 * dirty is protected by the object lock
2263 */
2264 SET_PAGE_DIRTY(copy_m, TRUE);
2265
2266 /*
2267 * The page is already ready for pageout:
2268 * not on pageout queues and busy.
2269 * Unlock everything except the
2270 * copy_object itself.
2271 */
2272 vm_object_unlock(object);
2273
2274 /*
2275 * Write the page to the copy-object,
2276 * flushing it from the kernel.
2277 */
2278 vm_pageout_initialize_page(copy_m);
2279
2280 /*
2281 * Since the pageout may have
2282 * temporarily dropped the
2283 * copy_object's lock, we
2284 * check whether we'll have
2285 * to deallocate the hard way.
2286 */
2287 if ((copy_object->shadow != object) || (copy_object->ref_count == 1)) {
2288 vm_object_unlock(copy_object);
2289 vm_object_deallocate(copy_object);
2290 vm_object_lock(object);
2291
2292 continue;
2293 }
2294 /*
2295 * Pick back up the old object's
2296 * lock. [It is safe to do so,
2297 * since it must be deeper in the
2298 * object tree.]
2299 */
2300 vm_object_lock(object);
2301 }
2302
2303 /*
2304 * Because we're pushing a page upward
2305 * in the object tree, we must restart
2306 * any faults that are waiting here.
2307 * [Note that this is an expansion of
2308 * PAGE_WAKEUP that uses the THREAD_RESTART
2309 * wait result]. Can't turn off the page's
2310 * busy bit because we're not done with it.
2311 */
2312 if (m->vmp_wanted) {
2313 m->vmp_wanted = FALSE;
2314 thread_wakeup_with_result((event_t) m, THREAD_RESTART);
2315 }
2316 }
2317 /*
2318 * The reference count on copy_object must be
2319 * at least 2: one for our extra reference,
2320 * and at least one from the outside world
2321 * (we checked that when we last locked
2322 * copy_object).
2323 */
2324 vm_object_lock_assert_exclusive(copy_object);
2325 copy_object->ref_count--;
2326 assert(copy_object->ref_count > 0);
2327
2328 VM_OBJ_RES_DECR(copy_object);
2329 vm_object_unlock(copy_object);
2330
2331 break;
2332 }
2333
2334 done:
2335 *result_page = m;
2336 *top_page = first_m;
2337
2338 if (m != VM_PAGE_NULL) {
2339 assert(VM_PAGE_OBJECT(m) == object);
2340
2341 retval = VM_FAULT_SUCCESS;
2342
2343 if (my_fault == DBG_PAGEIN_FAULT) {
2344 VM_PAGE_COUNT_AS_PAGEIN(m);
2345
2346 if (object->internal) {
2347 my_fault = DBG_PAGEIND_FAULT;
2348 } else {
2349 my_fault = DBG_PAGEINV_FAULT;
2350 }
2351
2352 /*
2353 * evaluate access pattern and update state
2354 * vm_fault_deactivate_behind depends on the
2355 * state being up to date
2356 */
2357 vm_fault_is_sequential(object, offset, fault_info->behavior);
2358 vm_fault_deactivate_behind(object, offset, fault_info->behavior);
2359 } else if (type_of_fault == NULL && my_fault == DBG_CACHE_HIT_FAULT) {
2360 /*
2361 * we weren't called from vm_fault, so handle the
2362 * accounting here for hits in the cache
2363 */
2364 if (m->vmp_clustered) {
2365 VM_PAGE_COUNT_AS_PAGEIN(m);
2366 VM_PAGE_CONSUME_CLUSTERED(m);
2367 }
2368 vm_fault_is_sequential(object, offset, fault_info->behavior);
2369 vm_fault_deactivate_behind(object, offset, fault_info->behavior);
2370 } else if (my_fault == DBG_COMPRESSOR_FAULT || my_fault == DBG_COMPRESSOR_SWAPIN_FAULT) {
2371 VM_STAT_DECOMPRESSIONS();
2372 }
2373 if (type_of_fault) {
2374 *type_of_fault = my_fault;
2375 }
2376 } else {
2377 retval = VM_FAULT_SUCCESS_NO_VM_PAGE;
2378 assert(first_m == VM_PAGE_NULL);
2379 assert(object == first_object);
2380 }
2381
2382 thread_interrupt_level(interruptible_state);
2383
2384 #if TRACEFAULTPAGE
2385 dbgTrace(0xBEEF001A, (unsigned int) VM_FAULT_SUCCESS, 0); /* (TEST/DEBUG) */
2386 #endif
2387 return retval;
2388
2389 backoff:
2390 thread_interrupt_level(interruptible_state);
2391
2392 if (wait_result == THREAD_INTERRUPTED) {
2393 return VM_FAULT_INTERRUPTED;
2394 }
2395 return VM_FAULT_RETRY;
2396
2397 #undef RELEASE_PAGE
2398 }
2399
2400
2401
2402 /*
2403 * CODE SIGNING:
2404 * When soft faulting a page, we have to validate the page if:
2405 * 1. the page is being mapped in user space
2406 * 2. the page hasn't already been found to be "tainted"
2407 * 3. the page belongs to a code-signed object
2408 * 4. the page has not been validated yet or has been mapped for write.
2409 */
2410 #define VM_FAULT_NEED_CS_VALIDATION(pmap, page, page_obj) \
2411 ((pmap) != kernel_pmap /*1*/ && \
2412 !(page)->vmp_cs_tainted /*2*/ && \
2413 (page_obj)->code_signed /*3*/ && \
2414 (!(page)->vmp_cs_validated || (page)->vmp_wpmapped /*4*/ ))
2415
2416
2417 /*
2418 * page queue lock must NOT be held
2419 * m->vmp_object must be locked
2420 *
2421 * NOTE: m->vmp_object could be locked "shared" only if we are called
2422 * from vm_fault() as part of a soft fault. If so, we must be
2423 * careful not to modify the VM object in any way that is not
2424 * legal under a shared lock...
2425 */
2426 extern int panic_on_cs_killed;
2427 extern int proc_selfpid(void);
2428 extern char *proc_name_address(void *p);
2429 unsigned long cs_enter_tainted_rejected = 0;
2430 unsigned long cs_enter_tainted_accepted = 0;
2431 kern_return_t
2432 vm_fault_enter(vm_page_t m,
2433 pmap_t pmap,
2434 vm_map_offset_t vaddr,
2435 vm_prot_t prot,
2436 vm_prot_t caller_prot,
2437 boolean_t wired,
2438 boolean_t change_wiring,
2439 vm_tag_t wire_tag,
2440 vm_object_fault_info_t fault_info,
2441 boolean_t *need_retry,
2442 int *type_of_fault)
2443 {
2444 kern_return_t kr, pe_result;
2445 boolean_t previously_pmapped = m->vmp_pmapped;
2446 boolean_t must_disconnect = 0;
2447 boolean_t map_is_switched, map_is_switch_protected;
2448 boolean_t cs_violation;
2449 int cs_enforcement_enabled;
2450 vm_prot_t fault_type;
2451 vm_object_t object;
2452 boolean_t no_cache = fault_info->no_cache;
2453 boolean_t cs_bypass = fault_info->cs_bypass;
2454 int pmap_options = fault_info->pmap_options;
2455
2456 fault_type = change_wiring ? VM_PROT_NONE : caller_prot;
2457 object = VM_PAGE_OBJECT(m);
2458
2459 vm_object_lock_assert_held(object);
2460
2461 #if KASAN
2462 if (pmap == kernel_pmap) {
2463 kasan_notify_address(vaddr, PAGE_SIZE);
2464 }
2465 #endif
2466
2467 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
2468
2469 if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
2470 assert(m->vmp_fictitious);
2471 return KERN_SUCCESS;
2472 }
2473
2474 if (*type_of_fault == DBG_ZERO_FILL_FAULT) {
2475 vm_object_lock_assert_exclusive(object);
2476 } else if ((fault_type & VM_PROT_WRITE) == 0 &&
2477 (!m->vmp_wpmapped
2478 #if VM_OBJECT_ACCESS_TRACKING
2479 || object->access_tracking
2480 #endif /* VM_OBJECT_ACCESS_TRACKING */
2481 )) {
2482 /*
2483 * This is not a "write" fault, so we
2484 * might not have taken the object lock
2485 * exclusively and we might not be able
2486 * to update the "wpmapped" bit in
2487 * vm_fault_enter().
2488 * Let's just grant read access to
2489 * the page for now and we'll
2490 * soft-fault again if we need write
2491 * access later...
2492 */
2493
2494 /* This had better not be a JIT page. */
2495 if (!pmap_has_prot_policy(prot)) {
2496 prot &= ~VM_PROT_WRITE;
2497 } else {
2498 assert(cs_bypass);
2499 }
2500 }
2501 if (m->vmp_pmapped == FALSE) {
2502 if (m->vmp_clustered) {
2503 if (*type_of_fault == DBG_CACHE_HIT_FAULT) {
2504 /*
2505 * found it in the cache, but this
2506 * is the first fault-in of the page (m->vmp_pmapped == FALSE)
2507 * so it must have come in as part of
2508 * a cluster... account 1 pagein against it
2509 */
2510 if (object->internal) {
2511 *type_of_fault = DBG_PAGEIND_FAULT;
2512 } else {
2513 *type_of_fault = DBG_PAGEINV_FAULT;
2514 }
2515
2516 VM_PAGE_COUNT_AS_PAGEIN(m);
2517 }
2518 VM_PAGE_CONSUME_CLUSTERED(m);
2519 }
2520 }
2521
2522 if (*type_of_fault != DBG_COW_FAULT) {
2523 DTRACE_VM2(as_fault, int, 1, (uint64_t *), NULL);
2524
2525 if (pmap == kernel_pmap) {
2526 DTRACE_VM2(kernel_asflt, int, 1, (uint64_t *), NULL);
2527 }
2528 }
2529
2530 /* Validate code signature if necessary. */
2531 if (!cs_bypass &&
2532 VM_FAULT_NEED_CS_VALIDATION(pmap, m, object)) {
2533 vm_object_lock_assert_exclusive(object);
2534
2535 if (m->vmp_cs_validated) {
2536 vm_cs_revalidates++;
2537 }
2538
2539 /* VM map is locked, so 1 ref will remain on VM object -
2540 * so no harm if vm_page_validate_cs drops the object lock */
2541
2542 #if PMAP_CS
2543 if (fault_info->pmap_cs_associated &&
2544 pmap_cs_enforced(pmap) &&
2545 !m->vmp_cs_validated &&
2546 !m->vmp_cs_tainted &&
2547 !m->vmp_cs_nx &&
2548 (prot & VM_PROT_EXECUTE) &&
2549 (caller_prot & VM_PROT_EXECUTE)) {
2550 /*
2551 * With pmap_cs, the pmap layer will validate the
2552 * code signature for any executable pmap mapping.
2553 * No need for us to validate this page too:
2554 * in pmap_cs we trust...
2555 */
2556 vm_cs_defer_to_pmap_cs++;
2557 } else {
2558 vm_cs_defer_to_pmap_cs_not++;
2559 vm_page_validate_cs(m);
2560 }
2561 #else /* PMAP_CS */
2562 vm_page_validate_cs(m);
2563 #endif /* PMAP_CS */
2564 }
2565
2566 #define page_immutable(m, prot) ((m)->vmp_cs_validated /*&& ((prot) & VM_PROT_EXECUTE)*/ )
2567 #define page_nx(m) ((m)->vmp_cs_nx)
2568
2569 map_is_switched = ((pmap != vm_map_pmap(current_task()->map)) &&
2570 (pmap == vm_map_pmap(current_thread()->map)));
2571 map_is_switch_protected = current_thread()->map->switch_protect;
2572
2573 /* If the map is switched, and is switch-protected, we must protect
2574 * some pages from being write-faulted: immutable pages because by
2575 * definition they may not be written, and executable pages because that
2576 * would provide a way to inject unsigned code.
2577 * If the page is immutable, we can simply return. However, we can't
2578 * immediately determine whether a page is executable anywhere. But,
2579 * we can disconnect it everywhere and remove the executable protection
2580 * from the current map. We do that below right before we do the
2581 * PMAP_ENTER.
2582 */
2583 if (pmap == kernel_pmap) {
2584 /* kernel fault: cs_process_enforcement() does not apply */
2585 cs_enforcement_enabled = 0;
2586 } else {
2587 cs_enforcement_enabled = cs_process_enforcement(NULL);
2588 }
2589
2590 if (cs_enforcement_enabled && map_is_switched &&
2591 map_is_switch_protected && page_immutable(m, prot) &&
2592 (prot & VM_PROT_WRITE)) {
2593 return KERN_CODESIGN_ERROR;
2594 }
2595
2596 if (cs_enforcement_enabled && page_nx(m) && (prot & VM_PROT_EXECUTE)) {
2597 if (cs_debug) {
2598 printf("page marked to be NX, not letting it be mapped EXEC\n");
2599 }
2600 return KERN_CODESIGN_ERROR;
2601 }
2602
2603 /* A page could be tainted, or pose a risk of being tainted later.
2604 * Check whether the receiving process wants it, and make it feel
2605 * the consequences (that hapens in cs_invalid_page()).
2606 * For CS Enforcement, two other conditions will
2607 * cause that page to be tainted as well:
2608 * - pmapping an unsigned page executable - this means unsigned code;
2609 * - writeable mapping of a validated page - the content of that page
2610 * can be changed without the kernel noticing, therefore unsigned
2611 * code can be created
2612 */
2613 if (cs_bypass) {
2614 /* code-signing is bypassed */
2615 cs_violation = FALSE;
2616 } else if (m->vmp_cs_tainted) {
2617 /* tainted page */
2618 cs_violation = TRUE;
2619 } else if (!cs_enforcement_enabled) {
2620 /* no further code-signing enforcement */
2621 cs_violation = FALSE;
2622 } else if (page_immutable(m, prot) &&
2623 ((prot & VM_PROT_WRITE) ||
2624 m->vmp_wpmapped)) {
2625 /*
2626 * The page should be immutable, but is in danger of being
2627 * modified.
2628 * This is the case where we want policy from the code
2629 * directory - is the page immutable or not? For now we have
2630 * to assume that code pages will be immutable, data pages not.
2631 * We'll assume a page is a code page if it has a code directory
2632 * and we fault for execution.
2633 * That is good enough since if we faulted the code page for
2634 * writing in another map before, it is wpmapped; if we fault
2635 * it for writing in this map later it will also be faulted for
2636 * executing at the same time; and if we fault for writing in
2637 * another map later, we will disconnect it from this pmap so
2638 * we'll notice the change.
2639 */
2640 cs_violation = TRUE;
2641 } else if (!m->vmp_cs_validated &&
2642 (prot & VM_PROT_EXECUTE)
2643 #if PMAP_CS
2644 /*
2645 * Executable pages will be validated by pmap_cs;
2646 * in pmap_cs we trust...
2647 * If pmap_cs is turned off, this is a code-signing
2648 * violation.
2649 */
2650 && !(pmap_cs_enforced(pmap))
2651 #endif /* PMAP_CS */
2652 ) {
2653 cs_violation = TRUE;
2654 } else {
2655 cs_violation = FALSE;
2656 }
2657
2658 if (cs_violation) {
2659 /* We will have a tainted page. Have to handle the special case
2660 * of a switched map now. If the map is not switched, standard
2661 * procedure applies - call cs_invalid_page().
2662 * If the map is switched, the real owner is invalid already.
2663 * There is no point in invalidating the switching process since
2664 * it will not be executing from the map. So we don't call
2665 * cs_invalid_page() in that case. */
2666 boolean_t reject_page, cs_killed;
2667 if (map_is_switched) {
2668 assert(pmap == vm_map_pmap(current_thread()->map));
2669 assert(!(prot & VM_PROT_WRITE) || (map_is_switch_protected == FALSE));
2670 reject_page = FALSE;
2671 } else {
2672 if (cs_debug > 5) {
2673 printf("vm_fault: signed: %s validate: %s tainted: %s wpmapped: %s prot: 0x%x\n",
2674 object->code_signed ? "yes" : "no",
2675 m->vmp_cs_validated ? "yes" : "no",
2676 m->vmp_cs_tainted ? "yes" : "no",
2677 m->vmp_wpmapped ? "yes" : "no",
2678 (int)prot);
2679 }
2680 reject_page = cs_invalid_page((addr64_t) vaddr, &cs_killed);
2681 }
2682
2683 if (reject_page) {
2684 /* reject the invalid page: abort the page fault */
2685 int pid;
2686 const char *procname;
2687 task_t task;
2688 vm_object_t file_object, shadow;
2689 vm_object_offset_t file_offset;
2690 char *pathname, *filename;
2691 vm_size_t pathname_len, filename_len;
2692 boolean_t truncated_path;
2693 #define __PATH_MAX 1024
2694 struct timespec mtime, cs_mtime;
2695 int shadow_depth;
2696 os_reason_t codesigning_exit_reason = OS_REASON_NULL;
2697
2698 kr = KERN_CODESIGN_ERROR;
2699 cs_enter_tainted_rejected++;
2700
2701 /* get process name and pid */
2702 procname = "?";
2703 task = current_task();
2704 pid = proc_selfpid();
2705 if (task->bsd_info != NULL) {
2706 procname = proc_name_address(task->bsd_info);
2707 }
2708
2709 /* get file's VM object */
2710 file_object = object;
2711 file_offset = m->vmp_offset;
2712 for (shadow = file_object->shadow,
2713 shadow_depth = 0;
2714 shadow != VM_OBJECT_NULL;
2715 shadow = file_object->shadow,
2716 shadow_depth++) {
2717 vm_object_lock_shared(shadow);
2718 if (file_object != object) {
2719 vm_object_unlock(file_object);
2720 }
2721 file_offset += file_object->vo_shadow_offset;
2722 file_object = shadow;
2723 }
2724
2725 mtime.tv_sec = 0;
2726 mtime.tv_nsec = 0;
2727 cs_mtime.tv_sec = 0;
2728 cs_mtime.tv_nsec = 0;
2729
2730 /* get file's pathname and/or filename */
2731 pathname = NULL;
2732 filename = NULL;
2733 pathname_len = 0;
2734 filename_len = 0;
2735 truncated_path = FALSE;
2736 /* no pager -> no file -> no pathname, use "<nil>" in that case */
2737 if (file_object->pager != NULL) {
2738 pathname = (char *)kalloc(__PATH_MAX * 2);
2739 if (pathname) {
2740 pathname[0] = '\0';
2741 pathname_len = __PATH_MAX;
2742 filename = pathname + pathname_len;
2743 filename_len = __PATH_MAX;
2744
2745 if (vnode_pager_get_object_name(file_object->pager,
2746 pathname,
2747 pathname_len,
2748 filename,
2749 filename_len,
2750 &truncated_path) == KERN_SUCCESS) {
2751 /* safety first... */
2752 pathname[__PATH_MAX - 1] = '\0';
2753 filename[__PATH_MAX - 1] = '\0';
2754
2755 vnode_pager_get_object_mtime(file_object->pager,
2756 &mtime,
2757 &cs_mtime);
2758 } else {
2759 kfree(pathname, __PATH_MAX * 2);
2760 pathname = NULL;
2761 filename = NULL;
2762 pathname_len = 0;
2763 filename_len = 0;
2764 truncated_path = FALSE;
2765 }
2766 }
2767 }
2768 printf("CODE SIGNING: process %d[%s]: "
2769 "rejecting invalid page at address 0x%llx "
2770 "from offset 0x%llx in file \"%s%s%s\" "
2771 "(cs_mtime:%lu.%ld %s mtime:%lu.%ld) "
2772 "(signed:%d validated:%d tainted:%d nx:%d "
2773 "wpmapped:%d dirty:%d depth:%d)\n",
2774 pid, procname, (addr64_t) vaddr,
2775 file_offset,
2776 (pathname ? pathname : "<nil>"),
2777 (truncated_path ? "/.../" : ""),
2778 (truncated_path ? filename : ""),
2779 cs_mtime.tv_sec, cs_mtime.tv_nsec,
2780 ((cs_mtime.tv_sec == mtime.tv_sec &&
2781 cs_mtime.tv_nsec == mtime.tv_nsec)
2782 ? "=="
2783 : "!="),
2784 mtime.tv_sec, mtime.tv_nsec,
2785 object->code_signed,
2786 m->vmp_cs_validated,
2787 m->vmp_cs_tainted,
2788 m->vmp_cs_nx,
2789 m->vmp_wpmapped,
2790 m->vmp_dirty,
2791 shadow_depth);
2792
2793 /*
2794 * We currently only generate an exit reason if cs_invalid_page directly killed a process. If cs_invalid_page
2795 * did not kill the process (more the case on desktop), vm_fault_enter will not satisfy the fault and whether the
2796 * process dies is dependent on whether there is a signal handler registered for SIGSEGV and how that handler
2797 * will deal with the segmentation fault.
2798 */
2799 if (cs_killed) {
2800 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXITREASON_CREATE) | DBG_FUNC_NONE,
2801 pid, OS_REASON_CODESIGNING, CODESIGNING_EXIT_REASON_INVALID_PAGE, 0, 0);
2802
2803 codesigning_exit_reason = os_reason_create(OS_REASON_CODESIGNING, CODESIGNING_EXIT_REASON_INVALID_PAGE);
2804 if (codesigning_exit_reason == NULL) {
2805 printf("vm_fault_enter: failed to allocate codesigning exit reason\n");
2806 } else {
2807 mach_vm_address_t data_addr = 0;
2808 struct codesigning_exit_reason_info *ceri = NULL;
2809 uint32_t reason_buffer_size_estimate = kcdata_estimate_required_buffer_size(1, sizeof(*ceri));
2810
2811 if (os_reason_alloc_buffer_noblock(codesigning_exit_reason, reason_buffer_size_estimate)) {
2812 printf("vm_fault_enter: failed to allocate buffer for codesigning exit reason\n");
2813 } else {
2814 if (KERN_SUCCESS == kcdata_get_memory_addr(&codesigning_exit_reason->osr_kcd_descriptor,
2815 EXIT_REASON_CODESIGNING_INFO, sizeof(*ceri), &data_addr)) {
2816 ceri = (struct codesigning_exit_reason_info *)data_addr;
2817 static_assert(__PATH_MAX == sizeof(ceri->ceri_pathname));
2818
2819 ceri->ceri_virt_addr = vaddr;
2820 ceri->ceri_file_offset = file_offset;
2821 if (pathname) {
2822 strncpy((char *)&ceri->ceri_pathname, pathname, sizeof(ceri->ceri_pathname));
2823 } else {
2824 ceri->ceri_pathname[0] = '\0';
2825 }
2826 if (filename) {
2827 strncpy((char *)&ceri->ceri_filename, filename, sizeof(ceri->ceri_filename));
2828 } else {
2829 ceri->ceri_filename[0] = '\0';
2830 }
2831 ceri->ceri_path_truncated = (truncated_path);
2832 ceri->ceri_codesig_modtime_secs = cs_mtime.tv_sec;
2833 ceri->ceri_codesig_modtime_nsecs = cs_mtime.tv_nsec;
2834 ceri->ceri_page_modtime_secs = mtime.tv_sec;
2835 ceri->ceri_page_modtime_nsecs = mtime.tv_nsec;
2836 ceri->ceri_object_codesigned = (object->code_signed);
2837 ceri->ceri_page_codesig_validated = (m->vmp_cs_validated);
2838 ceri->ceri_page_codesig_tainted = (m->vmp_cs_tainted);
2839 ceri->ceri_page_codesig_nx = (m->vmp_cs_nx);
2840 ceri->ceri_page_wpmapped = (m->vmp_wpmapped);
2841 ceri->ceri_page_slid = 0;
2842 ceri->ceri_page_dirty = (m->vmp_dirty);
2843 ceri->ceri_page_shadow_depth = shadow_depth;
2844 } else {
2845 #if DEBUG || DEVELOPMENT
2846 panic("vm_fault_enter: failed to allocate kcdata for codesigning exit reason");
2847 #else
2848 printf("vm_fault_enter: failed to allocate kcdata for codesigning exit reason\n");
2849 #endif /* DEBUG || DEVELOPMENT */
2850 /* Free the buffer */
2851 os_reason_alloc_buffer_noblock(codesigning_exit_reason, 0);
2852 }
2853 }
2854 }
2855
2856 set_thread_exit_reason(current_thread(), codesigning_exit_reason, FALSE);
2857 }
2858 if (panic_on_cs_killed &&
2859 object->object_is_shared_cache) {
2860 char *tainted_contents;
2861 vm_map_offset_t src_vaddr;
2862 src_vaddr = (vm_map_offset_t) phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(m) << PAGE_SHIFT);
2863 tainted_contents = kalloc(PAGE_SIZE);
2864 bcopy((const char *)src_vaddr, tainted_contents, PAGE_SIZE);
2865 printf("CODE SIGNING: tainted page %p phys 0x%x phystokv 0x%llx copied to %p\n", m, VM_PAGE_GET_PHYS_PAGE(m), (uint64_t)src_vaddr, tainted_contents);
2866 panic("CODE SIGNING: process %d[%s]: "
2867 "rejecting invalid page (phys#0x%x) at address 0x%llx "
2868 "from offset 0x%llx in file \"%s%s%s\" "
2869 "(cs_mtime:%lu.%ld %s mtime:%lu.%ld) "
2870 "(signed:%d validated:%d tainted:%d nx:%d"
2871 "wpmapped:%d dirty:%d depth:%d)\n",
2872 pid, procname,
2873 VM_PAGE_GET_PHYS_PAGE(m),
2874 (addr64_t) vaddr,
2875 file_offset,
2876 (pathname ? pathname : "<nil>"),
2877 (truncated_path ? "/.../" : ""),
2878 (truncated_path ? filename : ""),
2879 cs_mtime.tv_sec, cs_mtime.tv_nsec,
2880 ((cs_mtime.tv_sec == mtime.tv_sec &&
2881 cs_mtime.tv_nsec == mtime.tv_nsec)
2882 ? "=="
2883 : "!="),
2884 mtime.tv_sec, mtime.tv_nsec,
2885 object->code_signed,
2886 m->vmp_cs_validated,
2887 m->vmp_cs_tainted,
2888 m->vmp_cs_nx,
2889 m->vmp_wpmapped,
2890 m->vmp_dirty,
2891 shadow_depth);
2892 }
2893
2894 if (file_object != object) {
2895 vm_object_unlock(file_object);
2896 }
2897 if (pathname_len != 0) {
2898 kfree(pathname, __PATH_MAX * 2);
2899 pathname = NULL;
2900 filename = NULL;
2901 }
2902 } else {
2903 /* proceed with the invalid page */
2904 kr = KERN_SUCCESS;
2905 if (!m->vmp_cs_validated &&
2906 !object->code_signed) {
2907 /*
2908 * This page has not been (fully) validated but
2909 * does not belong to a code-signed object
2910 * so it should not be forcefully considered
2911 * as tainted.
2912 * We're just concerned about it here because
2913 * we've been asked to "execute" it but that
2914 * does not mean that it should cause other
2915 * accesses to fail.
2916 * This happens when a debugger sets a
2917 * breakpoint and we then execute code in
2918 * that page. Marking the page as "tainted"
2919 * would cause any inspection tool ("leaks",
2920 * "vmmap", "CrashReporter", ...) to get killed
2921 * due to code-signing violation on that page,
2922 * even though they're just reading it and not
2923 * executing from it.
2924 */
2925 } else {
2926 /*
2927 * Page might have been tainted before or not;
2928 * now it definitively is. If the page wasn't
2929 * tainted, we must disconnect it from all
2930 * pmaps later, to force existing mappings
2931 * through that code path for re-consideration
2932 * of the validity of that page.
2933 */
2934 must_disconnect = !m->vmp_cs_tainted;
2935 m->vmp_cs_tainted = TRUE;
2936 }
2937 cs_enter_tainted_accepted++;
2938 }
2939 if (kr != KERN_SUCCESS) {
2940 if (cs_debug) {
2941 printf("CODESIGNING: vm_fault_enter(0x%llx): "
2942 "*** INVALID PAGE ***\n",
2943 (long long)vaddr);
2944 }
2945 #if !SECURE_KERNEL
2946 if (cs_enforcement_panic) {
2947 panic("CODESIGNING: panicking on invalid page\n");
2948 }
2949 #endif
2950 }
2951 } else {
2952 /* proceed with the valid page */
2953 kr = KERN_SUCCESS;
2954 }
2955
2956 boolean_t page_queues_locked = FALSE;
2957 #define __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED() \
2958 MACRO_BEGIN \
2959 if (! page_queues_locked) { \
2960 page_queues_locked = TRUE; \
2961 vm_page_lockspin_queues(); \
2962 } \
2963 MACRO_END
2964 #define __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED() \
2965 MACRO_BEGIN \
2966 if (page_queues_locked) { \
2967 page_queues_locked = FALSE; \
2968 vm_page_unlock_queues(); \
2969 } \
2970 MACRO_END
2971
2972 /*
2973 * Hold queues lock to manipulate
2974 * the page queues. Change wiring
2975 * case is obvious.
2976 */
2977 assert((m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) || object != compressor_object);
2978
2979 #if CONFIG_BACKGROUND_QUEUE
2980 vm_page_update_background_state(m);
2981 #endif
2982 if (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
2983 /*
2984 * Compressor pages are neither wired
2985 * nor pageable and should never change.
2986 */
2987 assert(object == compressor_object);
2988 } else if (change_wiring) {
2989 __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED();
2990
2991 if (wired) {
2992 if (kr == KERN_SUCCESS) {
2993 vm_page_wire(m, wire_tag, TRUE);
2994 }
2995 } else {
2996 vm_page_unwire(m, TRUE);
2997 }
2998 /* we keep the page queues lock, if we need it later */
2999 } else {
3000 if (object->internal == TRUE) {
3001 /*
3002 * don't allow anonymous pages on
3003 * the speculative queues
3004 */
3005 no_cache = FALSE;
3006 }
3007 if (kr != KERN_SUCCESS) {
3008 __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED();
3009 vm_page_deactivate(m);
3010 /* we keep the page queues lock, if we need it later */
3011 } else if (((m->vmp_q_state == VM_PAGE_NOT_ON_Q) ||
3012 (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ||
3013 (m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) ||
3014 ((m->vmp_q_state != VM_PAGE_ON_THROTTLED_Q) && no_cache)) &&
3015 !VM_PAGE_WIRED(m)) {
3016 if (vm_page_local_q &&
3017 (*type_of_fault == DBG_COW_FAULT ||
3018 *type_of_fault == DBG_ZERO_FILL_FAULT)) {
3019 struct vpl *lq;
3020 uint32_t lid;
3021
3022 assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
3023
3024 __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED();
3025 vm_object_lock_assert_exclusive(object);
3026
3027 /*
3028 * we got a local queue to stuff this
3029 * new page on...
3030 * its safe to manipulate local and
3031 * local_id at this point since we're
3032 * behind an exclusive object lock and
3033 * the page is not on any global queue.
3034 *
3035 * we'll use the current cpu number to
3036 * select the queue note that we don't
3037 * need to disable preemption... we're
3038 * going to be behind the local queue's
3039 * lock to do the real work
3040 */
3041 lid = cpu_number();
3042
3043 lq = &vm_page_local_q[lid].vpl_un.vpl;
3044
3045 VPL_LOCK(&lq->vpl_lock);
3046
3047 vm_page_check_pageable_safe(m);
3048 vm_page_queue_enter(&lq->vpl_queue, m, vmp_pageq);
3049 m->vmp_q_state = VM_PAGE_ON_ACTIVE_LOCAL_Q;
3050 m->vmp_local_id = lid;
3051 lq->vpl_count++;
3052
3053 if (object->internal) {
3054 lq->vpl_internal_count++;
3055 } else {
3056 lq->vpl_external_count++;
3057 }
3058
3059 VPL_UNLOCK(&lq->vpl_lock);
3060
3061 if (lq->vpl_count > vm_page_local_q_soft_limit) {
3062 /*
3063 * we're beyond the soft limit
3064 * for the local queue
3065 * vm_page_reactivate_local will
3066 * 'try' to take the global page
3067 * queue lock... if it can't
3068 * that's ok... we'll let the
3069 * queue continue to grow up
3070 * to the hard limit... at that
3071 * point we'll wait for the
3072 * lock... once we've got the
3073 * lock, we'll transfer all of
3074 * the pages from the local
3075 * queue to the global active
3076 * queue
3077 */
3078 vm_page_reactivate_local(lid, FALSE, FALSE);
3079 }
3080 } else {
3081 __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED();
3082
3083 /*
3084 * test again now that we hold the
3085 * page queue lock
3086 */
3087 if (!VM_PAGE_WIRED(m)) {
3088 if (m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
3089 vm_page_queues_remove(m, FALSE);
3090
3091 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
3092 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_fault_reactivated, 1);
3093 }
3094
3095 if (!VM_PAGE_ACTIVE_OR_INACTIVE(m) ||
3096 no_cache) {
3097 /*
3098 * If this is a no_cache mapping
3099 * and the page has never been
3100 * mapped before or was
3101 * previously a no_cache page,
3102 * then we want to leave pages
3103 * in the speculative state so
3104 * that they can be readily
3105 * recycled if free memory runs
3106 * low. Otherwise the page is
3107 * activated as normal.
3108 */
3109
3110 if (no_cache &&
3111 (!previously_pmapped ||
3112 m->vmp_no_cache)) {
3113 m->vmp_no_cache = TRUE;
3114
3115 if (m->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) {
3116 vm_page_speculate(m, FALSE);
3117 }
3118 } else if (!VM_PAGE_ACTIVE_OR_INACTIVE(m)) {
3119 vm_page_activate(m);
3120 }
3121 }
3122 }
3123 /* we keep the page queues lock, if we need it later */
3124 }
3125 }
3126 }
3127 /* we're done with the page queues lock, if we ever took it */
3128 __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED();
3129
3130
3131 /* If we have a KERN_SUCCESS from the previous checks, we either have
3132 * a good page, or a tainted page that has been accepted by the process.
3133 * In both cases the page will be entered into the pmap.
3134 * If the page is writeable, we need to disconnect it from other pmaps
3135 * now so those processes can take note.
3136 */
3137 if (kr == KERN_SUCCESS) {
3138 /*
3139 * NOTE: we may only hold the vm_object lock SHARED
3140 * at this point, so we need the phys_page lock to
3141 * properly serialize updating the pmapped and
3142 * xpmapped bits
3143 */
3144 if ((prot & VM_PROT_EXECUTE) && !m->vmp_xpmapped) {
3145 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
3146
3147 pmap_lock_phys_page(phys_page);
3148 /*
3149 * go ahead and take the opportunity
3150 * to set 'pmapped' here so that we don't
3151 * need to grab this lock a 2nd time
3152 * just below
3153 */
3154 m->vmp_pmapped = TRUE;
3155
3156 if (!m->vmp_xpmapped) {
3157 m->vmp_xpmapped = TRUE;
3158
3159 pmap_unlock_phys_page(phys_page);
3160
3161 if (!object->internal) {
3162 OSAddAtomic(1, &vm_page_xpmapped_external_count);
3163 }
3164
3165 #if defined(__arm__) || defined(__arm64__)
3166 pmap_sync_page_data_phys(phys_page);
3167 #else
3168 if (object->internal &&
3169 object->pager != NULL) {
3170 /*
3171 * This page could have been
3172 * uncompressed by the
3173 * compressor pager and its
3174 * contents might be only in
3175 * the data cache.
3176 * Since it's being mapped for
3177 * "execute" for the fist time,
3178 * make sure the icache is in
3179 * sync.
3180 */
3181 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
3182 pmap_sync_page_data_phys(phys_page);
3183 }
3184 #endif
3185 } else {
3186 pmap_unlock_phys_page(phys_page);
3187 }
3188 } else {
3189 if (m->vmp_pmapped == FALSE) {
3190 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
3191
3192 pmap_lock_phys_page(phys_page);
3193 m->vmp_pmapped = TRUE;
3194 pmap_unlock_phys_page(phys_page);
3195 }
3196 }
3197
3198 if (fault_type & VM_PROT_WRITE) {
3199 if (m->vmp_wpmapped == FALSE) {
3200 vm_object_lock_assert_exclusive(object);
3201 if (!object->internal && object->pager) {
3202 task_update_logical_writes(current_task(), PAGE_SIZE, TASK_WRITE_DEFERRED, vnode_pager_lookup_vnode(object->pager));
3203 }
3204 m->vmp_wpmapped = TRUE;
3205 }
3206 if (must_disconnect) {
3207 /*
3208 * We can only get here
3209 * because of the CSE logic
3210 */
3211 assert(cs_enforcement_enabled);
3212 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
3213 /*
3214 * If we are faulting for a write, we can clear
3215 * the execute bit - that will ensure the page is
3216 * checked again before being executable, which
3217 * protects against a map switch.
3218 * This only happens the first time the page
3219 * gets tainted, so we won't get stuck here
3220 * to make an already writeable page executable.
3221 */
3222 if (!cs_bypass) {
3223 assert(!pmap_has_prot_policy(prot));
3224 prot &= ~VM_PROT_EXECUTE;
3225 }
3226 }
3227 }
3228 assert(VM_PAGE_OBJECT(m) == object);
3229
3230 #if VM_OBJECT_ACCESS_TRACKING
3231 if (object->access_tracking) {
3232 DTRACE_VM2(access_tracking, vm_map_offset_t, vaddr, int, fault_type);
3233 if (fault_type & VM_PROT_WRITE) {
3234 object->access_tracking_writes++;
3235 vm_object_access_tracking_writes++;
3236 } else {
3237 object->access_tracking_reads++;
3238 vm_object_access_tracking_reads++;
3239 }
3240 }
3241 #endif /* VM_OBJECT_ACCESS_TRACKING */
3242
3243
3244 #if PMAP_CS
3245 pmap_enter_retry:
3246 #endif
3247 /* Prevent a deadlock by not
3248 * holding the object lock if we need to wait for a page in
3249 * pmap_enter() - <rdar://problem/7138958> */
3250 PMAP_ENTER_OPTIONS(pmap, vaddr, m, prot, fault_type, 0,
3251 wired,
3252 pmap_options | PMAP_OPTIONS_NOWAIT,
3253 pe_result);
3254 #if PMAP_CS
3255 /*
3256 * Retry without execute permission if we encountered a codesigning
3257 * failure on a non-execute fault. This allows applications which
3258 * don't actually need to execute code to still map it for read access.
3259 */
3260 if ((pe_result == KERN_CODESIGN_ERROR) && pmap_cs_enforced(pmap) &&
3261 (prot & VM_PROT_EXECUTE) && !(caller_prot & VM_PROT_EXECUTE)) {
3262 prot &= ~VM_PROT_EXECUTE;
3263 goto pmap_enter_retry;
3264 }
3265 #endif
3266 #if __x86_64__
3267 if (pe_result == KERN_INVALID_ARGUMENT &&
3268 pmap == PMAP_NULL &&
3269 wired) {
3270 /*
3271 * Wiring a page in a pmap-less VM map:
3272 * VMware's "vmmon" kernel extension does this
3273 * to grab pages.
3274 * Let it proceed even though the PMAP_ENTER() failed.
3275 */
3276 pe_result = KERN_SUCCESS;
3277 }
3278 #endif /* __x86_64__ */
3279
3280 if (pe_result == KERN_RESOURCE_SHORTAGE) {
3281 if (need_retry) {
3282 /*
3283 * this will be non-null in the case where we hold the lock
3284 * on the top-object in this chain... we can't just drop
3285 * the lock on the object we're inserting the page into
3286 * and recall the PMAP_ENTER since we can still cause
3287 * a deadlock if one of the critical paths tries to
3288 * acquire the lock on the top-object and we're blocked
3289 * in PMAP_ENTER waiting for memory... our only recourse
3290 * is to deal with it at a higher level where we can
3291 * drop both locks.
3292 */
3293 *need_retry = TRUE;
3294 vm_pmap_enter_retried++;
3295 goto after_the_pmap_enter;
3296 }
3297 /* The nonblocking version of pmap_enter did not succeed.
3298 * and we don't need to drop other locks and retry
3299 * at the level above us, so
3300 * use the blocking version instead. Requires marking
3301 * the page busy and unlocking the object */
3302 boolean_t was_busy = m->vmp_busy;
3303
3304 vm_object_lock_assert_exclusive(object);
3305
3306 m->vmp_busy = TRUE;
3307 vm_object_unlock(object);
3308
3309 PMAP_ENTER_OPTIONS(pmap, vaddr, m, prot, fault_type,
3310 0, wired,
3311 pmap_options, pe_result);
3312
3313 assert(VM_PAGE_OBJECT(m) == object);
3314
3315 /* Take the object lock again. */
3316 vm_object_lock(object);
3317
3318 /* If the page was busy, someone else will wake it up.
3319 * Otherwise, we have to do it now. */
3320 assert(m->vmp_busy);
3321 if (!was_busy) {
3322 PAGE_WAKEUP_DONE(m);
3323 }
3324 vm_pmap_enter_blocked++;
3325 }
3326
3327 kr = pe_result;
3328 }
3329
3330 after_the_pmap_enter:
3331 return kr;
3332 }
3333
3334 void
3335 vm_pre_fault(vm_map_offset_t vaddr, vm_prot_t prot)
3336 {
3337 if (pmap_find_phys(current_map()->pmap, vaddr) == 0) {
3338 vm_fault(current_map(), /* map */
3339 vaddr, /* vaddr */
3340 prot, /* fault_type */
3341 FALSE, /* change_wiring */
3342 VM_KERN_MEMORY_NONE, /* tag - not wiring */
3343 THREAD_UNINT, /* interruptible */
3344 NULL, /* caller_pmap */
3345 0 /* caller_pmap_addr */);
3346 }
3347 }
3348
3349
3350 /*
3351 * Routine: vm_fault
3352 * Purpose:
3353 * Handle page faults, including pseudo-faults
3354 * used to change the wiring status of pages.
3355 * Returns:
3356 * Explicit continuations have been removed.
3357 * Implementation:
3358 * vm_fault and vm_fault_page save mucho state
3359 * in the moral equivalent of a closure. The state
3360 * structure is allocated when first entering vm_fault
3361 * and deallocated when leaving vm_fault.
3362 */
3363
3364 extern int _map_enter_debug;
3365 extern uint64_t get_current_unique_pid(void);
3366
3367 unsigned long vm_fault_collapse_total = 0;
3368 unsigned long vm_fault_collapse_skipped = 0;
3369
3370
3371 kern_return_t
3372 vm_fault_external(
3373 vm_map_t map,
3374 vm_map_offset_t vaddr,
3375 vm_prot_t fault_type,
3376 boolean_t change_wiring,
3377 int interruptible,
3378 pmap_t caller_pmap,
3379 vm_map_offset_t caller_pmap_addr)
3380 {
3381 return vm_fault_internal(map, vaddr, fault_type, change_wiring, vm_tag_bt(),
3382 interruptible, caller_pmap, caller_pmap_addr,
3383 NULL);
3384 }
3385
3386 kern_return_t
3387 vm_fault(
3388 vm_map_t map,
3389 vm_map_offset_t vaddr,
3390 vm_prot_t fault_type,
3391 boolean_t change_wiring,
3392 vm_tag_t wire_tag, /* if wiring must pass tag != VM_KERN_MEMORY_NONE */
3393 int interruptible,
3394 pmap_t caller_pmap,
3395 vm_map_offset_t caller_pmap_addr)
3396 {
3397 return vm_fault_internal(map, vaddr, fault_type, change_wiring, wire_tag,
3398 interruptible, caller_pmap, caller_pmap_addr,
3399 NULL);
3400 }
3401
3402 static boolean_t
3403 current_proc_is_privileged(void)
3404 {
3405 return csproc_get_platform_binary(current_proc());
3406 }
3407
3408 uint64_t vm_copied_on_read = 0;
3409
3410 kern_return_t
3411 vm_fault_internal(
3412 vm_map_t map,
3413 vm_map_offset_t vaddr,
3414 vm_prot_t caller_prot,
3415 boolean_t change_wiring,
3416 vm_tag_t wire_tag, /* if wiring must pass tag != VM_KERN_MEMORY_NONE */
3417 int interruptible,
3418 pmap_t caller_pmap,
3419 vm_map_offset_t caller_pmap_addr,
3420 ppnum_t *physpage_p)
3421 {
3422 vm_map_version_t version; /* Map version for verificiation */
3423 boolean_t wired; /* Should mapping be wired down? */
3424 vm_object_t object; /* Top-level object */
3425 vm_object_offset_t offset; /* Top-level offset */
3426 vm_prot_t prot; /* Protection for mapping */
3427 vm_object_t old_copy_object; /* Saved copy object */
3428 vm_page_t result_page; /* Result of vm_fault_page */
3429 vm_page_t top_page; /* Placeholder page */
3430 kern_return_t kr;
3431
3432 vm_page_t m; /* Fast access to result_page */
3433 kern_return_t error_code;
3434 vm_object_t cur_object;
3435 vm_object_t m_object = NULL;
3436 vm_object_offset_t cur_offset;
3437 vm_page_t cur_m;
3438 vm_object_t new_object;
3439 int type_of_fault;
3440 pmap_t pmap;
3441 wait_interrupt_t interruptible_state;
3442 vm_map_t real_map = map;
3443 vm_map_t original_map = map;
3444 boolean_t object_locks_dropped = FALSE;
3445 vm_prot_t fault_type;
3446 vm_prot_t original_fault_type;
3447 struct vm_object_fault_info fault_info = {};
3448 boolean_t need_collapse = FALSE;
3449 boolean_t need_retry = FALSE;
3450 boolean_t *need_retry_ptr = NULL;
3451 int object_lock_type = 0;
3452 int cur_object_lock_type;
3453 vm_object_t top_object = VM_OBJECT_NULL;
3454 vm_object_t written_on_object = VM_OBJECT_NULL;
3455 memory_object_t written_on_pager = NULL;
3456 vm_object_offset_t written_on_offset = 0;
3457 int throttle_delay;
3458 int compressed_count_delta;
3459 int grab_options;
3460 boolean_t need_copy;
3461 boolean_t need_copy_on_read;
3462 vm_map_offset_t trace_vaddr;
3463 vm_map_offset_t trace_real_vaddr;
3464 vm_map_offset_t real_vaddr;
3465 boolean_t resilient_media_retry = FALSE;
3466 vm_object_t resilient_media_object = VM_OBJECT_NULL;
3467 vm_object_offset_t resilient_media_offset = (vm_object_offset_t)-1;
3468
3469 real_vaddr = vaddr;
3470 trace_real_vaddr = vaddr;
3471 vaddr = vm_map_trunc_page(vaddr, PAGE_MASK);
3472
3473 if (map == kernel_map) {
3474 trace_vaddr = VM_KERNEL_ADDRHIDE(vaddr);
3475 trace_real_vaddr = VM_KERNEL_ADDRHIDE(trace_real_vaddr);
3476 } else {
3477 trace_vaddr = vaddr;
3478 }
3479
3480 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
3481 (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_START,
3482 ((uint64_t)trace_vaddr >> 32),
3483 trace_vaddr,
3484 (map == kernel_map),
3485 0,
3486 0);
3487
3488 if (get_preemption_level() != 0) {
3489 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
3490 (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END,
3491 ((uint64_t)trace_vaddr >> 32),
3492 trace_vaddr,
3493 KERN_FAILURE,
3494 0,
3495 0);
3496
3497 return KERN_FAILURE;
3498 }
3499
3500 thread_t cthread = current_thread();
3501 boolean_t rtfault = (cthread->sched_mode == TH_MODE_REALTIME);
3502 uint64_t fstart = 0;
3503
3504 if (rtfault) {
3505 fstart = mach_continuous_time();
3506 }
3507
3508 interruptible_state = thread_interrupt_level(interruptible);
3509
3510 fault_type = (change_wiring ? VM_PROT_NONE : caller_prot);
3511
3512 VM_STAT_INCR(faults);
3513 current_task()->faults++;
3514 original_fault_type = fault_type;
3515
3516 need_copy = FALSE;
3517 if (fault_type & VM_PROT_WRITE) {
3518 need_copy = TRUE;
3519 }
3520
3521 if (need_copy) {
3522 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3523 } else {
3524 object_lock_type = OBJECT_LOCK_SHARED;
3525 }
3526
3527 cur_object_lock_type = OBJECT_LOCK_SHARED;
3528
3529 if ((map == kernel_map) && (caller_prot & VM_PROT_WRITE)) {
3530 if (compressor_map) {
3531 if ((vaddr >= vm_map_min(compressor_map)) && (vaddr < vm_map_max(compressor_map))) {
3532 panic("Write fault on compressor map, va: %p type: %u bounds: %p->%p", (void *) vaddr, caller_prot, (void *) vm_map_min(compressor_map), (void *) vm_map_max(compressor_map));
3533 }
3534 }
3535 }
3536 RetryFault:
3537 assert(written_on_object == VM_OBJECT_NULL);
3538
3539 /*
3540 * assume we will hit a page in the cache
3541 * otherwise, explicitly override with
3542 * the real fault type once we determine it
3543 */
3544 type_of_fault = DBG_CACHE_HIT_FAULT;
3545
3546 /*
3547 * Find the backing store object and offset into
3548 * it to begin the search.
3549 */
3550 fault_type = original_fault_type;
3551 map = original_map;
3552 vm_map_lock_read(map);
3553
3554 if (resilient_media_retry) {
3555 /*
3556 * If we have to insert a fake zero-filled page to hide
3557 * a media failure to provide the real page, we need to
3558 * resolve any pending copy-on-write on this mapping.
3559 * VM_PROT_COPY tells vm_map_lookup_locked() to deal
3560 * with that even if this is not a "write" fault.
3561 */
3562 need_copy = TRUE;
3563 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3564 }
3565
3566 kr = vm_map_lookup_locked(&map, vaddr,
3567 (fault_type | (need_copy ? VM_PROT_COPY : 0)),
3568 object_lock_type, &version,
3569 &object, &offset, &prot, &wired,
3570 &fault_info,
3571 &real_map);
3572
3573 if (kr != KERN_SUCCESS) {
3574 vm_map_unlock_read(map);
3575 goto done;
3576 }
3577 pmap = real_map->pmap;
3578 fault_info.interruptible = interruptible;
3579 fault_info.stealth = FALSE;
3580 fault_info.io_sync = FALSE;
3581 fault_info.mark_zf_absent = FALSE;
3582 fault_info.batch_pmap_op = FALSE;
3583
3584 if (resilient_media_retry) {
3585 /*
3586 * We're retrying this fault after having detected a media
3587 * failure from a "resilient_media" mapping.
3588 * Check that the mapping is still pointing at the object
3589 * that just failed to provide a page.
3590 */
3591 assert(resilient_media_object != VM_OBJECT_NULL);
3592 assert(resilient_media_offset != (vm_object_offset_t)-1);
3593 if (object != VM_OBJECT_NULL &&
3594 object == resilient_media_object &&
3595 offset == resilient_media_offset &&
3596 fault_info.resilient_media) {
3597 /*
3598 * This mapping still points at the same object
3599 * and is still "resilient_media": proceed in
3600 * "recovery-from-media-failure" mode, where we'll
3601 * insert a zero-filled page in the top object.
3602 */
3603 // printf("RESILIENT_MEDIA %s:%d recovering for object %p offset 0x%llx\n", __FUNCTION__, __LINE__, object, offset);
3604 } else {
3605 /* not recovering: reset state */
3606 // printf("RESILIENT_MEDIA %s:%d no recovery resilient %d object %p/%p offset 0x%llx/0x%llx\n", __FUNCTION__, __LINE__, fault_info.resilient_media, object, resilient_media_object, offset, resilient_media_offset);
3607 resilient_media_retry = FALSE;
3608 /* release our extra reference on failed object */
3609 // printf("FBDP %s:%d resilient_media_object %p deallocate\n", __FUNCTION__, __LINE__, resilient_media_object);
3610 vm_object_deallocate(resilient_media_object);
3611 resilient_media_object = VM_OBJECT_NULL;
3612 resilient_media_offset = (vm_object_offset_t)-1;
3613 }
3614 } else {
3615 assert(resilient_media_object == VM_OBJECT_NULL);
3616 resilient_media_offset = (vm_object_offset_t)-1;
3617 }
3618
3619 /*
3620 * If the page is wired, we must fault for the current protection
3621 * value, to avoid further faults.
3622 */
3623 if (wired) {
3624 fault_type = prot | VM_PROT_WRITE;
3625 }
3626 if (wired || need_copy) {
3627 /*
3628 * since we're treating this fault as a 'write'
3629 * we must hold the top object lock exclusively
3630 */
3631 if (object_lock_type == OBJECT_LOCK_SHARED) {
3632 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3633
3634 if (vm_object_lock_upgrade(object) == FALSE) {
3635 /*
3636 * couldn't upgrade, so explictly
3637 * take the lock exclusively
3638 */
3639 vm_object_lock(object);
3640 }
3641 }
3642 }
3643
3644 #if VM_FAULT_CLASSIFY
3645 /*
3646 * Temporary data gathering code
3647 */
3648 vm_fault_classify(object, offset, fault_type);
3649 #endif
3650 /*
3651 * Fast fault code. The basic idea is to do as much as
3652 * possible while holding the map lock and object locks.
3653 * Busy pages are not used until the object lock has to
3654 * be dropped to do something (copy, zero fill, pmap enter).
3655 * Similarly, paging references aren't acquired until that
3656 * point, and object references aren't used.
3657 *
3658 * If we can figure out what to do
3659 * (zero fill, copy on write, pmap enter) while holding
3660 * the locks, then it gets done. Otherwise, we give up,
3661 * and use the original fault path (which doesn't hold
3662 * the map lock, and relies on busy pages).
3663 * The give up cases include:
3664 * - Have to talk to pager.
3665 * - Page is busy, absent or in error.
3666 * - Pager has locked out desired access.
3667 * - Fault needs to be restarted.
3668 * - Have to push page into copy object.
3669 *
3670 * The code is an infinite loop that moves one level down
3671 * the shadow chain each time. cur_object and cur_offset
3672 * refer to the current object being examined. object and offset
3673 * are the original object from the map. The loop is at the
3674 * top level if and only if object and cur_object are the same.
3675 *
3676 * Invariants: Map lock is held throughout. Lock is held on
3677 * original object and cur_object (if different) when
3678 * continuing or exiting loop.
3679 *
3680 */
3681
3682 #if defined(__arm64__)
3683 /*
3684 * Fail if reading an execute-only page in a
3685 * pmap that enforces execute-only protection.
3686 */
3687 if (fault_type == VM_PROT_READ &&
3688 (prot & VM_PROT_EXECUTE) &&
3689 !(prot & VM_PROT_READ) &&
3690 pmap_enforces_execute_only(pmap)) {
3691 vm_object_unlock(object);
3692 vm_map_unlock_read(map);
3693 if (real_map != map) {
3694 vm_map_unlock(real_map);
3695 }
3696 kr = KERN_PROTECTION_FAILURE;
3697 goto done;
3698 }
3699 #endif
3700
3701 /*
3702 * If this page is to be inserted in a copy delay object
3703 * for writing, and if the object has a copy, then the
3704 * copy delay strategy is implemented in the slow fault page.
3705 */
3706 if (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY &&
3707 object->copy != VM_OBJECT_NULL && (fault_type & VM_PROT_WRITE)) {
3708 goto handle_copy_delay;
3709 }
3710
3711 cur_object = object;
3712 cur_offset = offset;
3713
3714 grab_options = 0;
3715 #if CONFIG_SECLUDED_MEMORY
3716 if (object->can_grab_secluded) {
3717 grab_options |= VM_PAGE_GRAB_SECLUDED;
3718 }
3719 #endif /* CONFIG_SECLUDED_MEMORY */
3720
3721 while (TRUE) {
3722 if (!cur_object->pager_created &&
3723 cur_object->phys_contiguous) { /* superpage */
3724 break;
3725 }
3726
3727 if (cur_object->blocked_access) {
3728 /*
3729 * Access to this VM object has been blocked.
3730 * Let the slow path handle it.
3731 */
3732 break;
3733 }
3734
3735 m = vm_page_lookup(cur_object, cur_offset);
3736 m_object = NULL;
3737
3738 if (m != VM_PAGE_NULL) {
3739 m_object = cur_object;
3740
3741 if (m->vmp_busy) {
3742 wait_result_t result;
3743
3744 /*
3745 * in order to do the PAGE_ASSERT_WAIT, we must
3746 * have object that 'm' belongs to locked exclusively
3747 */
3748 if (object != cur_object) {
3749 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3750 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3751
3752 if (vm_object_lock_upgrade(cur_object) == FALSE) {
3753 /*
3754 * couldn't upgrade so go do a full retry
3755 * immediately since we can no longer be
3756 * certain about cur_object (since we
3757 * don't hold a reference on it)...
3758 * first drop the top object lock
3759 */
3760 vm_object_unlock(object);
3761
3762 vm_map_unlock_read(map);
3763 if (real_map != map) {
3764 vm_map_unlock(real_map);
3765 }
3766
3767 goto RetryFault;
3768 }
3769 }
3770 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3771 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3772
3773 if (vm_object_lock_upgrade(object) == FALSE) {
3774 /*
3775 * couldn't upgrade, so explictly take the lock
3776 * exclusively and go relookup the page since we
3777 * will have dropped the object lock and
3778 * a different thread could have inserted
3779 * a page at this offset
3780 * no need for a full retry since we're
3781 * at the top level of the object chain
3782 */
3783 vm_object_lock(object);
3784
3785 continue;
3786 }
3787 }
3788 if ((m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) && m_object->internal) {
3789 /*
3790 * m->vmp_busy == TRUE and the object is locked exclusively
3791 * if m->pageout_queue == TRUE after we acquire the
3792 * queues lock, we are guaranteed that it is stable on
3793 * the pageout queue and therefore reclaimable
3794 *
3795 * NOTE: this is only true for the internal pageout queue
3796 * in the compressor world
3797 */
3798 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
3799
3800 vm_page_lock_queues();
3801
3802 if (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
3803 vm_pageout_throttle_up(m);
3804 vm_page_unlock_queues();
3805
3806 PAGE_WAKEUP_DONE(m);
3807 goto reclaimed_from_pageout;
3808 }
3809 vm_page_unlock_queues();
3810 }
3811 if (object != cur_object) {
3812 vm_object_unlock(object);
3813 }
3814
3815 vm_map_unlock_read(map);
3816 if (real_map != map) {
3817 vm_map_unlock(real_map);
3818 }
3819
3820 result = PAGE_ASSERT_WAIT(m, interruptible);
3821
3822 vm_object_unlock(cur_object);
3823
3824 if (result == THREAD_WAITING) {
3825 result = thread_block(THREAD_CONTINUE_NULL);
3826
3827 counter(c_vm_fault_page_block_busy_kernel++);
3828 }
3829 if (result == THREAD_AWAKENED || result == THREAD_RESTART) {
3830 goto RetryFault;
3831 }
3832
3833 kr = KERN_ABORTED;
3834 goto done;
3835 }
3836 reclaimed_from_pageout:
3837 if (m->vmp_laundry) {
3838 if (object != cur_object) {
3839 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3840 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3841
3842 vm_object_unlock(object);
3843 vm_object_unlock(cur_object);
3844
3845 vm_map_unlock_read(map);
3846 if (real_map != map) {
3847 vm_map_unlock(real_map);
3848 }
3849
3850 goto RetryFault;
3851 }
3852 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3853 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3854
3855 if (vm_object_lock_upgrade(object) == FALSE) {
3856 /*
3857 * couldn't upgrade, so explictly take the lock
3858 * exclusively and go relookup the page since we
3859 * will have dropped the object lock and
3860 * a different thread could have inserted
3861 * a page at this offset
3862 * no need for a full retry since we're
3863 * at the top level of the object chain
3864 */
3865 vm_object_lock(object);
3866
3867 continue;
3868 }
3869 }
3870 vm_pageout_steal_laundry(m, FALSE);
3871 }
3872
3873 if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
3874 /*
3875 * Guard page: let the slow path deal with it
3876 */
3877 break;
3878 }
3879 if (m->vmp_unusual && (m->vmp_error || m->vmp_restart || m->vmp_private || m->vmp_absent)) {
3880 /*
3881 * Unusual case... let the slow path deal with it
3882 */
3883 break;
3884 }
3885 if (VM_OBJECT_PURGEABLE_FAULT_ERROR(m_object)) {
3886 if (object != cur_object) {
3887 vm_object_unlock(object);
3888 }
3889 vm_map_unlock_read(map);
3890 if (real_map != map) {
3891 vm_map_unlock(real_map);
3892 }
3893 vm_object_unlock(cur_object);
3894 kr = KERN_MEMORY_ERROR;
3895 goto done;
3896 }
3897 assert(m_object == VM_PAGE_OBJECT(m));
3898
3899 if (VM_FAULT_NEED_CS_VALIDATION(map->pmap, m, m_object) ||
3900 (physpage_p != NULL && (prot & VM_PROT_WRITE))) {
3901 upgrade_lock_and_retry:
3902 /*
3903 * We might need to validate this page
3904 * against its code signature, so we
3905 * want to hold the VM object exclusively.
3906 */
3907 if (object != cur_object) {
3908 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3909 vm_object_unlock(object);
3910 vm_object_unlock(cur_object);
3911
3912 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3913
3914 vm_map_unlock_read(map);
3915 if (real_map != map) {
3916 vm_map_unlock(real_map);
3917 }
3918
3919 goto RetryFault;
3920 }
3921 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3922 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3923
3924 if (vm_object_lock_upgrade(object) == FALSE) {
3925 /*
3926 * couldn't upgrade, so explictly take the lock
3927 * exclusively and go relookup the page since we
3928 * will have dropped the object lock and
3929 * a different thread could have inserted
3930 * a page at this offset
3931 * no need for a full retry since we're
3932 * at the top level of the object chain
3933 */
3934 vm_object_lock(object);
3935
3936 continue;
3937 }
3938 }
3939 }
3940 /*
3941 * Two cases of map in faults:
3942 * - At top level w/o copy object.
3943 * - Read fault anywhere.
3944 * --> must disallow write.
3945 */
3946
3947 if (object == cur_object && object->copy == VM_OBJECT_NULL) {
3948 goto FastPmapEnter;
3949 }
3950
3951 if (!need_copy &&
3952 !fault_info.no_copy_on_read &&
3953 cur_object != object &&
3954 !cur_object->internal &&
3955 !cur_object->pager_trusted &&
3956 vm_protect_privileged_from_untrusted &&
3957 !((prot & VM_PROT_EXECUTE) &&
3958 cur_object->code_signed &&
3959 cs_process_enforcement(NULL)) &&
3960 current_proc_is_privileged()) {
3961 /*
3962 * We're faulting on a page in "object" and
3963 * went down the shadow chain to "cur_object"
3964 * to find out that "cur_object"'s pager
3965 * is not "trusted", i.e. we can not trust it
3966 * to always return the same contents.
3967 * Since the target is a "privileged" process,
3968 * let's treat this as a copy-on-read fault, as
3969 * if it was a copy-on-write fault.
3970 * Once "object" gets a copy of this page, it
3971 * won't have to rely on "cur_object" to
3972 * provide the contents again.
3973 *
3974 * This is done by setting "need_copy" and
3975 * retrying the fault from the top with the
3976 * appropriate locking.
3977 *
3978 * Special case: if the mapping is executable
3979 * and the untrusted object is code-signed and
3980 * the process is "cs_enforced", we do not
3981 * copy-on-read because that would break
3982 * code-signing enforcement expectations (an
3983 * executable page must belong to a code-signed
3984 * object) and we can rely on code-signing
3985 * to re-validate the page if it gets evicted
3986 * and paged back in.
3987 */
3988 // printf("COPY-ON-READ %s:%d map %p va 0x%llx page %p object %p offset 0x%llx UNTRUSTED: need copy-on-read!\n", __FUNCTION__, __LINE__, map, (uint64_t)vaddr, m, VM_PAGE_OBJECT(m), m->vmp_offset);
3989 vm_copied_on_read++;
3990 need_copy = TRUE;
3991
3992 vm_object_unlock(object);
3993 vm_object_unlock(cur_object);
3994 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3995 vm_map_unlock_read(map);
3996 if (real_map != map) {
3997 vm_map_unlock(real_map);
3998 }
3999 goto RetryFault;
4000 }
4001
4002 if (!(fault_type & VM_PROT_WRITE) && !need_copy) {
4003 if (!pmap_has_prot_policy(prot)) {
4004 prot &= ~VM_PROT_WRITE;
4005 } else {
4006 /*
4007 * For a protection that the pmap cares
4008 * about, we must hand over the full
4009 * set of protections (so that the pmap
4010 * layer can apply any desired policy).
4011 * This means that cs_bypass must be
4012 * set, as this can force us to pass
4013 * RWX.
4014 */
4015 assert(fault_info.cs_bypass);
4016 }
4017
4018 if (object != cur_object) {
4019 /*
4020 * We still need to hold the top object
4021 * lock here to prevent a race between
4022 * a read fault (taking only "shared"
4023 * locks) and a write fault (taking
4024 * an "exclusive" lock on the top
4025 * object.
4026 * Otherwise, as soon as we release the
4027 * top lock, the write fault could
4028 * proceed and actually complete before
4029 * the read fault, and the copied page's
4030 * translation could then be overwritten
4031 * by the read fault's translation for
4032 * the original page.
4033 *
4034 * Let's just record what the top object
4035 * is and we'll release it later.
4036 */
4037 top_object = object;
4038
4039 /*
4040 * switch to the object that has the new page
4041 */
4042 object = cur_object;
4043 object_lock_type = cur_object_lock_type;
4044 }
4045 FastPmapEnter:
4046 assert(m_object == VM_PAGE_OBJECT(m));
4047
4048 /*
4049 * prepare for the pmap_enter...
4050 * object and map are both locked
4051 * m contains valid data
4052 * object == m->vmp_object
4053 * cur_object == NULL or it's been unlocked
4054 * no paging references on either object or cur_object
4055 */
4056 if (top_object != VM_OBJECT_NULL || object_lock_type != OBJECT_LOCK_EXCLUSIVE) {
4057 need_retry_ptr = &need_retry;
4058 } else {
4059 need_retry_ptr = NULL;
4060 }
4061
4062 if (caller_pmap) {
4063 kr = vm_fault_enter(m,
4064 caller_pmap,
4065 caller_pmap_addr,
4066 prot,
4067 caller_prot,
4068 wired,
4069 change_wiring,
4070 wire_tag,
4071 &fault_info,
4072 need_retry_ptr,
4073 &type_of_fault);
4074 } else {
4075 kr = vm_fault_enter(m,
4076 pmap,
4077 vaddr,
4078 prot,
4079 caller_prot,
4080 wired,
4081 change_wiring,
4082 wire_tag,
4083 &fault_info,
4084 need_retry_ptr,
4085 &type_of_fault);
4086 }
4087 {
4088 int event_code = 0;
4089
4090 if (m_object->internal) {
4091 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_INTERNAL));
4092 } else if (m_object->object_is_shared_cache) {
4093 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_SHAREDCACHE));
4094 } else {
4095 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_EXTERNAL));
4096 }
4097
4098 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, event_code, trace_real_vaddr, (fault_info.user_tag << 16) | (caller_prot << 8) | type_of_fault, m->vmp_offset, get_current_unique_pid(), 0);
4099 if (need_retry == FALSE) {
4100 KDBG_FILTERED(MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_FAST), get_current_unique_pid(), 0, 0, 0, 0);
4101 }
4102 DTRACE_VM6(real_fault, vm_map_offset_t, real_vaddr, vm_map_offset_t, m->vmp_offset, int, event_code, int, caller_prot, int, type_of_fault, int, fault_info.user_tag);
4103 }
4104 if (kr == KERN_SUCCESS &&
4105 physpage_p != NULL) {
4106 /* for vm_map_wire_and_extract() */
4107 *physpage_p = VM_PAGE_GET_PHYS_PAGE(m);
4108 if (prot & VM_PROT_WRITE) {
4109 vm_object_lock_assert_exclusive(m_object);
4110 m->vmp_dirty = TRUE;
4111 }
4112 }
4113
4114 if (top_object != VM_OBJECT_NULL) {
4115 /*
4116 * It's safe to drop the top object
4117 * now that we've done our
4118 * vm_fault_enter(). Any other fault
4119 * in progress for that virtual
4120 * address will either find our page
4121 * and translation or put in a new page
4122 * and translation.
4123 */
4124 vm_object_unlock(top_object);
4125 top_object = VM_OBJECT_NULL;
4126 }
4127
4128 if (need_collapse == TRUE) {
4129 vm_object_collapse(object, offset, TRUE);
4130 }
4131
4132 if (need_retry == FALSE &&
4133 (type_of_fault == DBG_PAGEIND_FAULT || type_of_fault == DBG_PAGEINV_FAULT || type_of_fault == DBG_CACHE_HIT_FAULT)) {
4134 /*
4135 * evaluate access pattern and update state
4136 * vm_fault_deactivate_behind depends on the
4137 * state being up to date
4138 */
4139 vm_fault_is_sequential(m_object, cur_offset, fault_info.behavior);
4140
4141 vm_fault_deactivate_behind(m_object, cur_offset, fault_info.behavior);
4142 }
4143 /*
4144 * That's it, clean up and return.
4145 */
4146 if (m->vmp_busy) {
4147 PAGE_WAKEUP_DONE(m);
4148 }
4149
4150 if (need_retry == FALSE && !m_object->internal && (fault_type & VM_PROT_WRITE)) {
4151 vm_object_paging_begin(m_object);
4152
4153 assert(written_on_object == VM_OBJECT_NULL);
4154 written_on_object = m_object;
4155 written_on_pager = m_object->pager;
4156 written_on_offset = m_object->paging_offset + m->vmp_offset;
4157 }
4158 vm_object_unlock(object);
4159
4160 vm_map_unlock_read(map);
4161 if (real_map != map) {
4162 vm_map_unlock(real_map);
4163 }
4164
4165 if (need_retry == TRUE) {
4166 /*
4167 * vm_fault_enter couldn't complete the PMAP_ENTER...
4168 * at this point we don't hold any locks so it's safe
4169 * to ask the pmap layer to expand the page table to
4170 * accommodate this mapping... once expanded, we'll
4171 * re-drive the fault which should result in vm_fault_enter
4172 * being able to successfully enter the mapping this time around
4173 */
4174 (void)pmap_enter_options(
4175 pmap, vaddr, 0, 0, 0, 0, 0,
4176 PMAP_OPTIONS_NOENTER, NULL);
4177
4178 need_retry = FALSE;
4179 goto RetryFault;
4180 }
4181 goto done;
4182 }
4183 /*
4184 * COPY ON WRITE FAULT
4185 */
4186 assert(object_lock_type == OBJECT_LOCK_EXCLUSIVE);
4187
4188 /*
4189 * If objects match, then
4190 * object->copy must not be NULL (else control
4191 * would be in previous code block), and we
4192 * have a potential push into the copy object
4193 * with which we can't cope with here.
4194 */
4195 if (cur_object == object) {
4196 /*
4197 * must take the slow path to
4198 * deal with the copy push
4199 */
4200 break;
4201 }
4202
4203 /*
4204 * This is now a shadow based copy on write
4205 * fault -- it requires a copy up the shadow
4206 * chain.
4207 */
4208 assert(m_object == VM_PAGE_OBJECT(m));
4209
4210 if ((cur_object_lock_type == OBJECT_LOCK_SHARED) &&
4211 VM_FAULT_NEED_CS_VALIDATION(NULL, m, m_object)) {
4212 goto upgrade_lock_and_retry;
4213 }
4214
4215 /*
4216 * Allocate a page in the original top level
4217 * object. Give up if allocate fails. Also
4218 * need to remember current page, as it's the
4219 * source of the copy.
4220 *
4221 * at this point we hold locks on both
4222 * object and cur_object... no need to take
4223 * paging refs or mark pages BUSY since
4224 * we don't drop either object lock until
4225 * the page has been copied and inserted
4226 */
4227 cur_m = m;
4228 m = vm_page_grab_options(grab_options);
4229 m_object = NULL;
4230
4231 if (m == VM_PAGE_NULL) {
4232 /*
4233 * no free page currently available...
4234 * must take the slow path
4235 */
4236 break;
4237 }
4238 /*
4239 * Now do the copy. Mark the source page busy...
4240 *
4241 * NOTE: This code holds the map lock across
4242 * the page copy.
4243 */
4244 vm_page_copy(cur_m, m);
4245 vm_page_insert(m, object, offset);
4246 m_object = object;
4247 SET_PAGE_DIRTY(m, FALSE);
4248
4249 /*
4250 * Now cope with the source page and object
4251 */
4252 if (object->ref_count > 1 && cur_m->vmp_pmapped) {
4253 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(cur_m));
4254 }
4255
4256 if (cur_m->vmp_clustered) {
4257 VM_PAGE_COUNT_AS_PAGEIN(cur_m);
4258 VM_PAGE_CONSUME_CLUSTERED(cur_m);
4259 vm_fault_is_sequential(cur_object, cur_offset, fault_info.behavior);
4260 }
4261 need_collapse = TRUE;
4262
4263 if (!cur_object->internal &&
4264 cur_object->copy_strategy == MEMORY_OBJECT_COPY_DELAY) {
4265 /*
4266 * The object from which we've just
4267 * copied a page is most probably backed
4268 * by a vnode. We don't want to waste too
4269 * much time trying to collapse the VM objects
4270 * and create a bottleneck when several tasks
4271 * map the same file.
4272 */
4273 if (cur_object->copy == object) {
4274 /*
4275 * Shared mapping or no COW yet.
4276 * We can never collapse a copy
4277 * object into its backing object.
4278 */
4279 need_collapse = FALSE;
4280 } else if (cur_object->copy == object->shadow &&
4281 object->shadow->resident_page_count == 0) {
4282 /*
4283 * Shared mapping after a COW occurred.
4284 */
4285 need_collapse = FALSE;
4286 }
4287 }
4288 vm_object_unlock(cur_object);
4289
4290 if (need_collapse == FALSE) {
4291 vm_fault_collapse_skipped++;
4292 }
4293 vm_fault_collapse_total++;
4294
4295 type_of_fault = DBG_COW_FAULT;
4296 VM_STAT_INCR(cow_faults);
4297 DTRACE_VM2(cow_fault, int, 1, (uint64_t *), NULL);
4298 current_task()->cow_faults++;
4299
4300 goto FastPmapEnter;
4301 } else {
4302 /*
4303 * No page at cur_object, cur_offset... m == NULL
4304 */
4305 if (cur_object->pager_created) {
4306 int compressor_external_state = VM_EXTERNAL_STATE_UNKNOWN;
4307
4308 if (MUST_ASK_PAGER(cur_object, cur_offset, compressor_external_state) == TRUE) {
4309 int my_fault_type;
4310 int c_flags = C_DONT_BLOCK;
4311 boolean_t insert_cur_object = FALSE;
4312
4313 /*
4314 * May have to talk to a pager...
4315 * if so, take the slow path by
4316 * doing a 'break' from the while (TRUE) loop
4317 *
4318 * external_state will only be set to VM_EXTERNAL_STATE_EXISTS
4319 * if the compressor is active and the page exists there
4320 */
4321 if (compressor_external_state != VM_EXTERNAL_STATE_EXISTS) {
4322 break;
4323 }
4324
4325 if (map == kernel_map || real_map == kernel_map) {
4326 /*
4327 * can't call into the compressor with the kernel_map
4328 * lock held, since the compressor may try to operate
4329 * on the kernel map in order to return an empty c_segment
4330 */
4331 break;
4332 }
4333 if (object != cur_object) {
4334 if (fault_type & VM_PROT_WRITE) {
4335 c_flags |= C_KEEP;
4336 } else {
4337 insert_cur_object = TRUE;
4338 }
4339 }
4340 if (insert_cur_object == TRUE) {
4341 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
4342 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4343
4344 if (vm_object_lock_upgrade(cur_object) == FALSE) {
4345 /*
4346 * couldn't upgrade so go do a full retry
4347 * immediately since we can no longer be
4348 * certain about cur_object (since we
4349 * don't hold a reference on it)...
4350 * first drop the top object lock
4351 */
4352 vm_object_unlock(object);
4353
4354 vm_map_unlock_read(map);
4355 if (real_map != map) {
4356 vm_map_unlock(real_map);
4357 }
4358
4359 goto RetryFault;
4360 }
4361 }
4362 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
4363 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4364
4365 if (object != cur_object) {
4366 /*
4367 * we can't go for the upgrade on the top
4368 * lock since the upgrade may block waiting
4369 * for readers to drain... since we hold
4370 * cur_object locked at this point, waiting
4371 * for the readers to drain would represent
4372 * a lock order inversion since the lock order
4373 * for objects is the reference order in the
4374 * shadown chain
4375 */
4376 vm_object_unlock(object);
4377 vm_object_unlock(cur_object);
4378
4379 vm_map_unlock_read(map);
4380 if (real_map != map) {
4381 vm_map_unlock(real_map);
4382 }
4383
4384 goto RetryFault;
4385 }
4386 if (vm_object_lock_upgrade(object) == FALSE) {
4387 /*
4388 * couldn't upgrade, so explictly take the lock
4389 * exclusively and go relookup the page since we
4390 * will have dropped the object lock and
4391 * a different thread could have inserted
4392 * a page at this offset
4393 * no need for a full retry since we're
4394 * at the top level of the object chain
4395 */
4396 vm_object_lock(object);
4397
4398 continue;
4399 }
4400 }
4401 m = vm_page_grab_options(grab_options);
4402 m_object = NULL;
4403
4404 if (m == VM_PAGE_NULL) {
4405 /*
4406 * no free page currently available...
4407 * must take the slow path
4408 */
4409 break;
4410 }
4411
4412 /*
4413 * The object is and remains locked
4414 * so no need to take a
4415 * "paging_in_progress" reference.
4416 */
4417 boolean_t shared_lock;
4418 if ((object == cur_object &&
4419 object_lock_type == OBJECT_LOCK_EXCLUSIVE) ||
4420 (object != cur_object &&
4421 cur_object_lock_type == OBJECT_LOCK_EXCLUSIVE)) {
4422 shared_lock = FALSE;
4423 } else {
4424 shared_lock = TRUE;
4425 }
4426
4427 kr = vm_compressor_pager_get(
4428 cur_object->pager,
4429 (cur_offset +
4430 cur_object->paging_offset),
4431 VM_PAGE_GET_PHYS_PAGE(m),
4432 &my_fault_type,
4433 c_flags,
4434 &compressed_count_delta);
4435
4436 vm_compressor_pager_count(
4437 cur_object->pager,
4438 compressed_count_delta,
4439 shared_lock,
4440 cur_object);
4441
4442 if (kr != KERN_SUCCESS) {
4443 vm_page_release(m, FALSE);
4444 m = VM_PAGE_NULL;
4445 break;
4446 }
4447 m->vmp_dirty = TRUE;
4448
4449 /*
4450 * If the object is purgeable, its
4451 * owner's purgeable ledgers will be
4452 * updated in vm_page_insert() but the
4453 * page was also accounted for in a
4454 * "compressed purgeable" ledger, so
4455 * update that now.
4456 */
4457 if (object != cur_object &&
4458 !insert_cur_object) {
4459 /*
4460 * We're not going to insert
4461 * the decompressed page into
4462 * the object it came from.
4463 *
4464 * We're dealing with a
4465 * copy-on-write fault on
4466 * "object".
4467 * We're going to decompress
4468 * the page directly into the
4469 * target "object" while
4470 * keepin the compressed
4471 * page for "cur_object", so
4472 * no ledger update in that
4473 * case.
4474 */
4475 } else if (((cur_object->purgable ==
4476 VM_PURGABLE_DENY) &&
4477 (!cur_object->vo_ledger_tag)) ||
4478 (cur_object->vo_owner ==
4479 NULL)) {
4480 /*
4481 * "cur_object" is not purgeable
4482 * and is not ledger-taged, or
4483 * there's no owner for it,
4484 * so no owner's ledgers to
4485 * update.
4486 */
4487 } else {
4488 /*
4489 * One less compressed
4490 * purgeable/tagged page for
4491 * cur_object's owner.
4492 */
4493 vm_object_owner_compressed_update(
4494 cur_object,
4495 -1);
4496 }
4497
4498 if (insert_cur_object) {
4499 vm_page_insert(m, cur_object, cur_offset);
4500 m_object = cur_object;
4501 } else {
4502 vm_page_insert(m, object, offset);
4503 m_object = object;
4504 }
4505
4506 if ((m_object->wimg_bits & VM_WIMG_MASK) != VM_WIMG_USE_DEFAULT) {
4507 /*
4508 * If the page is not cacheable,
4509 * we can't let its contents
4510 * linger in the data cache
4511 * after the decompression.
4512 */
4513 pmap_sync_page_attributes_phys(VM_PAGE_GET_PHYS_PAGE(m));
4514 }
4515
4516 type_of_fault = my_fault_type;
4517
4518 VM_STAT_DECOMPRESSIONS();
4519
4520 if (cur_object != object) {
4521 if (insert_cur_object) {
4522 top_object = object;
4523 /*
4524 * switch to the object that has the new page
4525 */
4526 object = cur_object;
4527 object_lock_type = cur_object_lock_type;
4528 } else {
4529 vm_object_unlock(cur_object);
4530 cur_object = object;
4531 }
4532 }
4533 goto FastPmapEnter;
4534 }
4535 /*
4536 * existence map present and indicates
4537 * that the pager doesn't have this page
4538 */
4539 }
4540 if (cur_object->shadow == VM_OBJECT_NULL ||
4541 resilient_media_retry) {
4542 /*
4543 * Zero fill fault. Page gets
4544 * inserted into the original object.
4545 */
4546 if (cur_object->shadow_severed ||
4547 VM_OBJECT_PURGEABLE_FAULT_ERROR(cur_object) ||
4548 cur_object == compressor_object ||
4549 cur_object == kernel_object ||
4550 cur_object == vm_submap_object) {
4551 if (object != cur_object) {
4552 vm_object_unlock(cur_object);
4553 }
4554 vm_object_unlock(object);
4555
4556 vm_map_unlock_read(map);
4557 if (real_map != map) {
4558 vm_map_unlock(real_map);
4559 }
4560
4561 kr = KERN_MEMORY_ERROR;
4562 goto done;
4563 }
4564 if (cur_object != object) {
4565 vm_object_unlock(cur_object);
4566
4567 cur_object = object;
4568 }
4569 if (object_lock_type == OBJECT_LOCK_SHARED) {
4570 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4571
4572 if (vm_object_lock_upgrade(object) == FALSE) {
4573 /*
4574 * couldn't upgrade so do a full retry on the fault
4575 * since we dropped the object lock which
4576 * could allow another thread to insert
4577 * a page at this offset
4578 */
4579 vm_map_unlock_read(map);
4580 if (real_map != map) {
4581 vm_map_unlock(real_map);
4582 }
4583
4584 goto RetryFault;
4585 }
4586 }
4587 if (!object->internal) {
4588 panic("%s:%d should not zero-fill page at offset 0x%llx in external object %p", __FUNCTION__, __LINE__, (uint64_t)offset, object);
4589 }
4590 m = vm_page_alloc(object, offset);
4591 m_object = NULL;
4592
4593 if (m == VM_PAGE_NULL) {
4594 /*
4595 * no free page currently available...
4596 * must take the slow path
4597 */
4598 break;
4599 }
4600 m_object = object;
4601
4602 /*
4603 * Now zero fill page...
4604 * the page is probably going to
4605 * be written soon, so don't bother
4606 * to clear the modified bit
4607 *
4608 * NOTE: This code holds the map
4609 * lock across the zero fill.
4610 */
4611 type_of_fault = vm_fault_zero_page(m, map->no_zero_fill);
4612
4613 goto FastPmapEnter;
4614 }
4615 /*
4616 * On to the next level in the shadow chain
4617 */
4618 cur_offset += cur_object->vo_shadow_offset;
4619 new_object = cur_object->shadow;
4620
4621 /*
4622 * take the new_object's lock with the indicated state
4623 */
4624 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
4625 vm_object_lock_shared(new_object);
4626 } else {
4627 vm_object_lock(new_object);
4628 }
4629
4630 if (cur_object != object) {
4631 vm_object_unlock(cur_object);
4632 }
4633
4634 cur_object = new_object;
4635
4636 continue;
4637 }
4638 }
4639 /*
4640 * Cleanup from fast fault failure. Drop any object
4641 * lock other than original and drop map lock.
4642 */
4643 if (object != cur_object) {
4644 vm_object_unlock(cur_object);
4645 }
4646
4647 /*
4648 * must own the object lock exclusively at this point
4649 */
4650 if (object_lock_type == OBJECT_LOCK_SHARED) {
4651 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4652
4653 if (vm_object_lock_upgrade(object) == FALSE) {
4654 /*
4655 * couldn't upgrade, so explictly
4656 * take the lock exclusively
4657 * no need to retry the fault at this
4658 * point since "vm_fault_page" will
4659 * completely re-evaluate the state
4660 */
4661 vm_object_lock(object);
4662 }
4663 }
4664
4665 handle_copy_delay:
4666 vm_map_unlock_read(map);
4667 if (real_map != map) {
4668 vm_map_unlock(real_map);
4669 }
4670
4671 if (__improbable(object == compressor_object ||
4672 object == kernel_object ||
4673 object == vm_submap_object)) {
4674 /*
4675 * These objects are explicitly managed and populated by the
4676 * kernel. The virtual ranges backed by these objects should
4677 * either have wired pages or "holes" that are not supposed to
4678 * be accessed at all until they get explicitly populated.
4679 * We should never have to resolve a fault on a mapping backed
4680 * by one of these VM objects and providing a zero-filled page
4681 * would be wrong here, so let's fail the fault and let the
4682 * caller crash or recover.
4683 */
4684 vm_object_unlock(object);
4685 kr = KERN_MEMORY_ERROR;
4686 goto done;
4687 }
4688
4689 assert(object != compressor_object);
4690 assert(object != kernel_object);
4691 assert(object != vm_submap_object);
4692
4693 if (resilient_media_retry) {
4694 /*
4695 * We could get here if we failed to get a free page
4696 * to zero-fill and had to take the slow path again.
4697 * Reset our "recovery-from-failed-media" state.
4698 */
4699 assert(resilient_media_object != VM_OBJECT_NULL);
4700 assert(resilient_media_offset != (vm_object_offset_t)-1);
4701 /* release our extra reference on failed object */
4702 // printf("FBDP %s:%d resilient_media_object %p deallocate\n", __FUNCTION__, __LINE__, resilient_media_object);
4703 vm_object_deallocate(resilient_media_object);
4704 resilient_media_object = VM_OBJECT_NULL;
4705 resilient_media_offset = (vm_object_offset_t)-1;
4706 resilient_media_retry = FALSE;
4707 }
4708
4709 /*
4710 * Make a reference to this object to
4711 * prevent its disposal while we are messing with
4712 * it. Once we have the reference, the map is free
4713 * to be diddled. Since objects reference their
4714 * shadows (and copies), they will stay around as well.
4715 */
4716 vm_object_reference_locked(object);
4717 vm_object_paging_begin(object);
4718
4719 set_thread_pagein_error(cthread, 0);
4720 error_code = 0;
4721
4722 result_page = VM_PAGE_NULL;
4723 kr = vm_fault_page(object, offset, fault_type,
4724 (change_wiring && !wired),
4725 FALSE, /* page not looked up */
4726 &prot, &result_page, &top_page,
4727 &type_of_fault,
4728 &error_code, map->no_zero_fill,
4729 FALSE, &fault_info);
4730
4731 /*
4732 * if kr != VM_FAULT_SUCCESS, then the paging reference
4733 * has been dropped and the object unlocked... the ref_count
4734 * is still held
4735 *
4736 * if kr == VM_FAULT_SUCCESS, then the paging reference
4737 * is still held along with the ref_count on the original object
4738 *
4739 * the object is returned locked with a paging reference
4740 *
4741 * if top_page != NULL, then it's BUSY and the
4742 * object it belongs to has a paging reference
4743 * but is returned unlocked
4744 */
4745 if (kr != VM_FAULT_SUCCESS &&
4746 kr != VM_FAULT_SUCCESS_NO_VM_PAGE) {
4747 if (kr == VM_FAULT_MEMORY_ERROR &&
4748 fault_info.resilient_media) {
4749 assertf(object->internal, "object %p", object);
4750 /*
4751 * This fault failed but the mapping was
4752 * "media resilient", so we'll retry the fault in
4753 * recovery mode to get a zero-filled page in the
4754 * top object.
4755 * Keep the reference on the failing object so
4756 * that we can check that the mapping is still
4757 * pointing to it when we retry the fault.
4758 */
4759 // printf("RESILIENT_MEDIA %s:%d: object %p offset 0x%llx recover from media error 0x%x kr 0x%x top_page %p result_page %p\n", __FUNCTION__, __LINE__, object, offset, error_code, kr, top_page, result_page);
4760 assert(!resilient_media_retry); /* no double retry */
4761 assert(resilient_media_object == VM_OBJECT_NULL);
4762 assert(resilient_media_offset == (vm_object_offset_t)-1);
4763 resilient_media_retry = TRUE;
4764 resilient_media_object = object;
4765 resilient_media_offset = offset;
4766 // printf("FBDP %s:%d resilient_media_object %p offset 0x%llx kept reference\n", __FUNCTION__, __LINE__, resilient_media_object, resilient_mmedia_offset);
4767 goto RetryFault;
4768 } else {
4769 /*
4770 * we didn't succeed, lose the object reference
4771 * immediately.
4772 */
4773 vm_object_deallocate(object);
4774 object = VM_OBJECT_NULL; /* no longer valid */
4775 }
4776
4777 /*
4778 * See why we failed, and take corrective action.
4779 */
4780 switch (kr) {
4781 case VM_FAULT_MEMORY_SHORTAGE:
4782 if (vm_page_wait((change_wiring) ?
4783 THREAD_UNINT :
4784 THREAD_ABORTSAFE)) {
4785 goto RetryFault;
4786 }
4787 /*
4788 * fall thru
4789 */
4790 case VM_FAULT_INTERRUPTED:
4791 kr = KERN_ABORTED;
4792 goto done;
4793 case VM_FAULT_RETRY:
4794 goto RetryFault;
4795 case VM_FAULT_MEMORY_ERROR:
4796 if (error_code) {
4797 kr = error_code;
4798 } else {
4799 kr = KERN_MEMORY_ERROR;
4800 }
4801 goto done;
4802 default:
4803 panic("vm_fault: unexpected error 0x%x from "
4804 "vm_fault_page()\n", kr);
4805 }
4806 }
4807 m = result_page;
4808 m_object = NULL;
4809
4810 if (m != VM_PAGE_NULL) {
4811 m_object = VM_PAGE_OBJECT(m);
4812 assert((change_wiring && !wired) ?
4813 (top_page == VM_PAGE_NULL) :
4814 ((top_page == VM_PAGE_NULL) == (m_object == object)));
4815 }
4816
4817 /*
4818 * What to do with the resulting page from vm_fault_page
4819 * if it doesn't get entered into the physical map:
4820 */
4821 #define RELEASE_PAGE(m) \
4822 MACRO_BEGIN \
4823 PAGE_WAKEUP_DONE(m); \
4824 if ( !VM_PAGE_PAGEABLE(m)) { \
4825 vm_page_lockspin_queues(); \
4826 if ( !VM_PAGE_PAGEABLE(m)) \
4827 vm_page_activate(m); \
4828 vm_page_unlock_queues(); \
4829 } \
4830 MACRO_END
4831
4832
4833 object_locks_dropped = FALSE;
4834 /*
4835 * We must verify that the maps have not changed
4836 * since our last lookup. vm_map_verify() needs the
4837 * map lock (shared) but we are holding object locks.
4838 * So we do a try_lock() first and, if that fails, we
4839 * drop the object locks and go in for the map lock again.
4840 */
4841 if (!vm_map_try_lock_read(original_map)) {
4842 if (m != VM_PAGE_NULL) {
4843 old_copy_object = m_object->copy;
4844 vm_object_unlock(m_object);
4845 } else {
4846 old_copy_object = VM_OBJECT_NULL;
4847 vm_object_unlock(object);
4848 }
4849
4850 object_locks_dropped = TRUE;
4851
4852 vm_map_lock_read(original_map);
4853 }
4854
4855 if ((map != original_map) || !vm_map_verify(map, &version)) {
4856 if (object_locks_dropped == FALSE) {
4857 if (m != VM_PAGE_NULL) {
4858 old_copy_object = m_object->copy;
4859 vm_object_unlock(m_object);
4860 } else {
4861 old_copy_object = VM_OBJECT_NULL;
4862 vm_object_unlock(object);
4863 }
4864
4865 object_locks_dropped = TRUE;
4866 }
4867
4868 /*
4869 * no object locks are held at this point
4870 */
4871 vm_object_t retry_object;
4872 vm_object_offset_t retry_offset;
4873 vm_prot_t retry_prot;
4874
4875 /*
4876 * To avoid trying to write_lock the map while another
4877 * thread has it read_locked (in vm_map_pageable), we
4878 * do not try for write permission. If the page is
4879 * still writable, we will get write permission. If it
4880 * is not, or has been marked needs_copy, we enter the
4881 * mapping without write permission, and will merely
4882 * take another fault.
4883 */
4884 map = original_map;
4885
4886 kr = vm_map_lookup_locked(&map, vaddr,
4887 fault_type & ~VM_PROT_WRITE,
4888 OBJECT_LOCK_EXCLUSIVE, &version,
4889 &retry_object, &retry_offset, &retry_prot,
4890 &wired,
4891 &fault_info,
4892 &real_map);
4893 pmap = real_map->pmap;
4894
4895 if (kr != KERN_SUCCESS) {
4896 vm_map_unlock_read(map);
4897
4898 if (m != VM_PAGE_NULL) {
4899 assert(VM_PAGE_OBJECT(m) == m_object);
4900
4901 /*
4902 * retake the lock so that
4903 * we can drop the paging reference
4904 * in vm_fault_cleanup and do the
4905 * PAGE_WAKEUP_DONE in RELEASE_PAGE
4906 */
4907 vm_object_lock(m_object);
4908
4909 RELEASE_PAGE(m);
4910
4911 vm_fault_cleanup(m_object, top_page);
4912 } else {
4913 /*
4914 * retake the lock so that
4915 * we can drop the paging reference
4916 * in vm_fault_cleanup
4917 */
4918 vm_object_lock(object);
4919
4920 vm_fault_cleanup(object, top_page);
4921 }
4922 vm_object_deallocate(object);
4923
4924 goto done;
4925 }
4926 vm_object_unlock(retry_object);
4927
4928 if ((retry_object != object) || (retry_offset != offset)) {
4929 vm_map_unlock_read(map);
4930 if (real_map != map) {
4931 vm_map_unlock(real_map);
4932 }
4933
4934 if (m != VM_PAGE_NULL) {
4935 assert(VM_PAGE_OBJECT(m) == m_object);
4936
4937 /*
4938 * retake the lock so that
4939 * we can drop the paging reference
4940 * in vm_fault_cleanup and do the
4941 * PAGE_WAKEUP_DONE in RELEASE_PAGE
4942 */
4943 vm_object_lock(m_object);
4944
4945 RELEASE_PAGE(m);
4946
4947 vm_fault_cleanup(m_object, top_page);
4948 } else {
4949 /*
4950 * retake the lock so that
4951 * we can drop the paging reference
4952 * in vm_fault_cleanup
4953 */
4954 vm_object_lock(object);
4955
4956 vm_fault_cleanup(object, top_page);
4957 }
4958 vm_object_deallocate(object);
4959
4960 goto RetryFault;
4961 }
4962 /*
4963 * Check whether the protection has changed or the object
4964 * has been copied while we left the map unlocked.
4965 */
4966 if (pmap_has_prot_policy(retry_prot)) {
4967 /* If the pmap layer cares, pass the full set. */
4968 prot = retry_prot;
4969 } else {
4970 prot &= retry_prot;
4971 }
4972 }
4973
4974 if (object_locks_dropped == TRUE) {
4975 if (m != VM_PAGE_NULL) {
4976 vm_object_lock(m_object);
4977
4978 if (m_object->copy != old_copy_object) {
4979 /*
4980 * The copy object changed while the top-level object
4981 * was unlocked, so take away write permission.
4982 */
4983 assert(!pmap_has_prot_policy(prot));
4984 prot &= ~VM_PROT_WRITE;
4985 }
4986 } else {
4987 vm_object_lock(object);
4988 }
4989
4990 object_locks_dropped = FALSE;
4991 }
4992
4993 if (!need_copy &&
4994 !fault_info.no_copy_on_read &&
4995 m != VM_PAGE_NULL &&
4996 VM_PAGE_OBJECT(m) != object &&
4997 !VM_PAGE_OBJECT(m)->pager_trusted &&
4998 vm_protect_privileged_from_untrusted &&
4999 !((prot & VM_PROT_EXECUTE) &&
5000 VM_PAGE_OBJECT(m)->code_signed &&
5001 cs_process_enforcement(NULL)) &&
5002 current_proc_is_privileged()) {
5003 /*
5004 * We found the page we want in an "untrusted" VM object
5005 * down the shadow chain. Since the target is "privileged"
5006 * we want to perform a copy-on-read of that page, so that the
5007 * mapped object gets a stable copy and does not have to
5008 * rely on the "untrusted" object to provide the same
5009 * contents if the page gets reclaimed and has to be paged
5010 * in again later on.
5011 *
5012 * Special case: if the mapping is executable and the untrusted
5013 * object is code-signed and the process is "cs_enforced", we
5014 * do not copy-on-read because that would break code-signing
5015 * enforcement expectations (an executable page must belong
5016 * to a code-signed object) and we can rely on code-signing
5017 * to re-validate the page if it gets evicted and paged back in.
5018 */
5019 // printf("COPY-ON-READ %s:%d map %p vaddr 0x%llx obj %p offset 0x%llx found page %p (obj %p offset 0x%llx) UNTRUSTED -> need copy-on-read\n", __FUNCTION__, __LINE__, map, (uint64_t)vaddr, object, offset, m, VM_PAGE_OBJECT(m), m->vmp_offset);
5020 vm_copied_on_read++;
5021 need_copy_on_read = TRUE;
5022 need_copy = TRUE;
5023 } else {
5024 need_copy_on_read = FALSE;
5025 }
5026
5027 /*
5028 * If we want to wire down this page, but no longer have
5029 * adequate permissions, we must start all over.
5030 * If we decided to copy-on-read, we must also start all over.
5031 */
5032 if ((wired && (fault_type != (prot | VM_PROT_WRITE))) ||
5033 need_copy_on_read) {
5034 vm_map_unlock_read(map);
5035 if (real_map != map) {
5036 vm_map_unlock(real_map);
5037 }
5038
5039 if (m != VM_PAGE_NULL) {
5040 assert(VM_PAGE_OBJECT(m) == m_object);
5041
5042 RELEASE_PAGE(m);
5043
5044 vm_fault_cleanup(m_object, top_page);
5045 } else {
5046 vm_fault_cleanup(object, top_page);
5047 }
5048
5049 vm_object_deallocate(object);
5050
5051 goto RetryFault;
5052 }
5053 if (m != VM_PAGE_NULL) {
5054 /*
5055 * Put this page into the physical map.
5056 * We had to do the unlock above because pmap_enter
5057 * may cause other faults. The page may be on
5058 * the pageout queues. If the pageout daemon comes
5059 * across the page, it will remove it from the queues.
5060 */
5061 if (caller_pmap) {
5062 kr = vm_fault_enter(m,
5063 caller_pmap,
5064 caller_pmap_addr,
5065 prot,
5066 caller_prot,
5067 wired,
5068 change_wiring,
5069 wire_tag,
5070 &fault_info,
5071 NULL,
5072 &type_of_fault);
5073 } else {
5074 kr = vm_fault_enter(m,
5075 pmap,
5076 vaddr,
5077 prot,
5078 caller_prot,
5079 wired,
5080 change_wiring,
5081 wire_tag,
5082 &fault_info,
5083 NULL,
5084 &type_of_fault);
5085 }
5086 assert(VM_PAGE_OBJECT(m) == m_object);
5087
5088 {
5089 int event_code = 0;
5090
5091 if (m_object->internal) {
5092 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_INTERNAL));
5093 } else if (m_object->object_is_shared_cache) {
5094 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_SHAREDCACHE));
5095 } else {
5096 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_EXTERNAL));
5097 }
5098
5099 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, event_code, trace_real_vaddr, (fault_info.user_tag << 16) | (caller_prot << 8) | type_of_fault, m->vmp_offset, get_current_unique_pid(), 0);
5100 KDBG_FILTERED(MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_SLOW), get_current_unique_pid(), 0, 0, 0, 0);
5101
5102 DTRACE_VM6(real_fault, vm_map_offset_t, real_vaddr, vm_map_offset_t, m->vmp_offset, int, event_code, int, caller_prot, int, type_of_fault, int, fault_info.user_tag);
5103 }
5104 if (kr != KERN_SUCCESS) {
5105 /* abort this page fault */
5106 vm_map_unlock_read(map);
5107 if (real_map != map) {
5108 vm_map_unlock(real_map);
5109 }
5110 PAGE_WAKEUP_DONE(m);
5111 vm_fault_cleanup(m_object, top_page);
5112 vm_object_deallocate(object);
5113 goto done;
5114 }
5115 if (physpage_p != NULL) {
5116 /* for vm_map_wire_and_extract() */
5117 *physpage_p = VM_PAGE_GET_PHYS_PAGE(m);
5118 if (prot & VM_PROT_WRITE) {
5119 vm_object_lock_assert_exclusive(m_object);
5120 m->vmp_dirty = TRUE;
5121 }
5122 }
5123 } else {
5124 vm_map_entry_t entry;
5125 vm_map_offset_t laddr;
5126 vm_map_offset_t ldelta, hdelta;
5127
5128 /*
5129 * do a pmap block mapping from the physical address
5130 * in the object
5131 */
5132
5133 if (real_map != map) {
5134 vm_map_unlock(real_map);
5135 }
5136
5137 if (original_map != map) {
5138 vm_map_unlock_read(map);
5139 vm_map_lock_read(original_map);
5140 map = original_map;
5141 }
5142 real_map = map;
5143
5144 laddr = vaddr;
5145 hdelta = 0xFFFFF000;
5146 ldelta = 0xFFFFF000;
5147
5148 while (vm_map_lookup_entry(map, laddr, &entry)) {
5149 if (ldelta > (laddr - entry->vme_start)) {
5150 ldelta = laddr - entry->vme_start;
5151 }
5152 if (hdelta > (entry->vme_end - laddr)) {
5153 hdelta = entry->vme_end - laddr;
5154 }
5155 if (entry->is_sub_map) {
5156 laddr = ((laddr - entry->vme_start)
5157 + VME_OFFSET(entry));
5158 vm_map_lock_read(VME_SUBMAP(entry));
5159
5160 if (map != real_map) {
5161 vm_map_unlock_read(map);
5162 }
5163 if (entry->use_pmap) {
5164 vm_map_unlock_read(real_map);
5165 real_map = VME_SUBMAP(entry);
5166 }
5167 map = VME_SUBMAP(entry);
5168 } else {
5169 break;
5170 }
5171 }
5172
5173 if (vm_map_lookup_entry(map, laddr, &entry) &&
5174 (VME_OBJECT(entry) != NULL) &&
5175 (VME_OBJECT(entry) == object)) {
5176 int superpage;
5177
5178 if (!object->pager_created &&
5179 object->phys_contiguous &&
5180 VME_OFFSET(entry) == 0 &&
5181 (entry->vme_end - entry->vme_start == object->vo_size) &&
5182 VM_MAP_PAGE_ALIGNED(entry->vme_start, (object->vo_size - 1))) {
5183 superpage = VM_MEM_SUPERPAGE;
5184 } else {
5185 superpage = 0;
5186 }
5187
5188 if (superpage && physpage_p) {
5189 /* for vm_map_wire_and_extract() */
5190 *physpage_p = (ppnum_t)
5191 ((((vm_map_offset_t)
5192 object->vo_shadow_offset)
5193 + VME_OFFSET(entry)
5194 + (laddr - entry->vme_start))
5195 >> PAGE_SHIFT);
5196 }
5197
5198 if (caller_pmap) {
5199 /*
5200 * Set up a block mapped area
5201 */
5202 assert((uint32_t)((ldelta + hdelta) >> PAGE_SHIFT) == ((ldelta + hdelta) >> PAGE_SHIFT));
5203 kr = pmap_map_block(caller_pmap,
5204 (addr64_t)(caller_pmap_addr - ldelta),
5205 (ppnum_t)((((vm_map_offset_t) (VME_OBJECT(entry)->vo_shadow_offset)) +
5206 VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT),
5207 (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot,
5208 (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0);
5209
5210 if (kr != KERN_SUCCESS) {
5211 goto cleanup;
5212 }
5213 } else {
5214 /*
5215 * Set up a block mapped area
5216 */
5217 assert((uint32_t)((ldelta + hdelta) >> PAGE_SHIFT) == ((ldelta + hdelta) >> PAGE_SHIFT));
5218 kr = pmap_map_block(real_map->pmap,
5219 (addr64_t)(vaddr - ldelta),
5220 (ppnum_t)((((vm_map_offset_t)(VME_OBJECT(entry)->vo_shadow_offset)) +
5221 VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT),
5222 (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot,
5223 (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0);
5224
5225 if (kr != KERN_SUCCESS) {
5226 goto cleanup;
5227 }
5228 }
5229 }
5230 }
5231
5232 /*
5233 * Success
5234 */
5235 kr = KERN_SUCCESS;
5236
5237 /*
5238 * TODO: could most of the done cases just use cleanup?
5239 */
5240 cleanup:
5241 /*
5242 * Unlock everything, and return
5243 */
5244 vm_map_unlock_read(map);
5245 if (real_map != map) {
5246 vm_map_unlock(real_map);
5247 }
5248
5249 if (m != VM_PAGE_NULL) {
5250 assert(VM_PAGE_OBJECT(m) == m_object);
5251
5252 if (!m_object->internal && (fault_type & VM_PROT_WRITE)) {
5253 vm_object_paging_begin(m_object);
5254
5255 assert(written_on_object == VM_OBJECT_NULL);
5256 written_on_object = m_object;
5257 written_on_pager = m_object->pager;
5258 written_on_offset = m_object->paging_offset + m->vmp_offset;
5259 }
5260 PAGE_WAKEUP_DONE(m);
5261
5262 vm_fault_cleanup(m_object, top_page);
5263 } else {
5264 vm_fault_cleanup(object, top_page);
5265 }
5266
5267 vm_object_deallocate(object);
5268
5269 #undef RELEASE_PAGE
5270
5271 done:
5272 thread_interrupt_level(interruptible_state);
5273
5274 if (resilient_media_object != VM_OBJECT_NULL) {
5275 assert(resilient_media_retry);
5276 assert(resilient_media_offset != (vm_object_offset_t)-1);
5277 /* release extra reference on failed object */
5278 // printf("FBDP %s:%d resilient_media_object %p deallocate\n", __FUNCTION__, __LINE__, resilient_media_object);
5279 vm_object_deallocate(resilient_media_object);
5280 resilient_media_object = VM_OBJECT_NULL;
5281 resilient_media_offset = (vm_object_offset_t)-1;
5282 resilient_media_retry = FALSE;
5283 }
5284 assert(!resilient_media_retry);
5285
5286 /*
5287 * Only I/O throttle on faults which cause a pagein/swapin.
5288 */
5289 if ((type_of_fault == DBG_PAGEIND_FAULT) || (type_of_fault == DBG_PAGEINV_FAULT) || (type_of_fault == DBG_COMPRESSOR_SWAPIN_FAULT)) {
5290 throttle_lowpri_io(1);
5291 } else {
5292 if (kr == KERN_SUCCESS && type_of_fault != DBG_CACHE_HIT_FAULT && type_of_fault != DBG_GUARD_FAULT) {
5293 if ((throttle_delay = vm_page_throttled(TRUE))) {
5294 if (vm_debug_events) {
5295 if (type_of_fault == DBG_COMPRESSOR_FAULT) {
5296 VM_DEBUG_EVENT(vmf_compressordelay, VMF_COMPRESSORDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
5297 } else if (type_of_fault == DBG_COW_FAULT) {
5298 VM_DEBUG_EVENT(vmf_cowdelay, VMF_COWDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
5299 } else {
5300 VM_DEBUG_EVENT(vmf_zfdelay, VMF_ZFDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
5301 }
5302 }
5303 delay(throttle_delay);
5304 }
5305 }
5306 }
5307
5308 if (written_on_object) {
5309 vnode_pager_dirtied(written_on_pager, written_on_offset, written_on_offset + PAGE_SIZE_64);
5310
5311 vm_object_lock(written_on_object);
5312 vm_object_paging_end(written_on_object);
5313 vm_object_unlock(written_on_object);
5314
5315 written_on_object = VM_OBJECT_NULL;
5316 }
5317
5318 if (rtfault) {
5319 vm_record_rtfault(cthread, fstart, trace_vaddr, type_of_fault);
5320 }
5321
5322 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
5323 (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END,
5324 ((uint64_t)trace_vaddr >> 32),
5325 trace_vaddr,
5326 kr,
5327 type_of_fault,
5328 0);
5329
5330 return kr;
5331 }
5332
5333 /*
5334 * vm_fault_wire:
5335 *
5336 * Wire down a range of virtual addresses in a map.
5337 */
5338 kern_return_t
5339 vm_fault_wire(
5340 vm_map_t map,
5341 vm_map_entry_t entry,
5342 vm_prot_t prot,
5343 vm_tag_t wire_tag,
5344 pmap_t pmap,
5345 vm_map_offset_t pmap_addr,
5346 ppnum_t *physpage_p)
5347 {
5348 vm_map_offset_t va;
5349 vm_map_offset_t end_addr = entry->vme_end;
5350 kern_return_t rc;
5351
5352 assert(entry->in_transition);
5353
5354 if ((VME_OBJECT(entry) != NULL) &&
5355 !entry->is_sub_map &&
5356 VME_OBJECT(entry)->phys_contiguous) {
5357 return KERN_SUCCESS;
5358 }
5359
5360 /*
5361 * Inform the physical mapping system that the
5362 * range of addresses may not fault, so that
5363 * page tables and such can be locked down as well.
5364 */
5365
5366 pmap_pageable(pmap, pmap_addr,
5367 pmap_addr + (end_addr - entry->vme_start), FALSE);
5368
5369 /*
5370 * We simulate a fault to get the page and enter it
5371 * in the physical map.
5372 */
5373
5374 for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
5375 rc = vm_fault_wire_fast(map, va, prot, wire_tag, entry, pmap,
5376 pmap_addr + (va - entry->vme_start),
5377 physpage_p);
5378 if (rc != KERN_SUCCESS) {
5379 rc = vm_fault_internal(map, va, prot, TRUE, wire_tag,
5380 ((pmap == kernel_pmap)
5381 ? THREAD_UNINT
5382 : THREAD_ABORTSAFE),
5383 pmap,
5384 (pmap_addr +
5385 (va - entry->vme_start)),
5386 physpage_p);
5387 DTRACE_VM2(softlock, int, 1, (uint64_t *), NULL);
5388 }
5389
5390 if (rc != KERN_SUCCESS) {
5391 struct vm_map_entry tmp_entry = *entry;
5392
5393 /* unwire wired pages */
5394 tmp_entry.vme_end = va;
5395 vm_fault_unwire(map,
5396 &tmp_entry, FALSE, pmap, pmap_addr);
5397
5398 return rc;
5399 }
5400 }
5401 return KERN_SUCCESS;
5402 }
5403
5404 /*
5405 * vm_fault_unwire:
5406 *
5407 * Unwire a range of virtual addresses in a map.
5408 */
5409 void
5410 vm_fault_unwire(
5411 vm_map_t map,
5412 vm_map_entry_t entry,
5413 boolean_t deallocate,
5414 pmap_t pmap,
5415 vm_map_offset_t pmap_addr)
5416 {
5417 vm_map_offset_t va;
5418 vm_map_offset_t end_addr = entry->vme_end;
5419 vm_object_t object;
5420 struct vm_object_fault_info fault_info = {};
5421 unsigned int unwired_pages;
5422
5423 object = (entry->is_sub_map) ? VM_OBJECT_NULL : VME_OBJECT(entry);
5424
5425 /*
5426 * If it's marked phys_contiguous, then vm_fault_wire() didn't actually
5427 * do anything since such memory is wired by default. So we don't have
5428 * anything to undo here.
5429 */
5430
5431 if (object != VM_OBJECT_NULL && object->phys_contiguous) {
5432 return;
5433 }
5434
5435 fault_info.interruptible = THREAD_UNINT;
5436 fault_info.behavior = entry->behavior;
5437 fault_info.user_tag = VME_ALIAS(entry);
5438 if (entry->iokit_acct ||
5439 (!entry->is_sub_map && !entry->use_pmap)) {
5440 fault_info.pmap_options |= PMAP_OPTIONS_ALT_ACCT;
5441 }
5442 fault_info.lo_offset = VME_OFFSET(entry);
5443 fault_info.hi_offset = (entry->vme_end - entry->vme_start) + VME_OFFSET(entry);
5444 fault_info.no_cache = entry->no_cache;
5445 fault_info.stealth = TRUE;
5446
5447 unwired_pages = 0;
5448
5449 /*
5450 * Since the pages are wired down, we must be able to
5451 * get their mappings from the physical map system.
5452 */
5453
5454 for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
5455 if (object == VM_OBJECT_NULL) {
5456 if (pmap) {
5457 pmap_change_wiring(pmap,
5458 pmap_addr + (va - entry->vme_start), FALSE);
5459 }
5460 (void) vm_fault(map, va, VM_PROT_NONE,
5461 TRUE, VM_KERN_MEMORY_NONE, THREAD_UNINT, pmap, pmap_addr);
5462 } else {
5463 vm_prot_t prot;
5464 vm_page_t result_page;
5465 vm_page_t top_page;
5466 vm_object_t result_object;
5467 vm_fault_return_t result;
5468
5469 /* cap cluster size at maximum UPL size */
5470 upl_size_t cluster_size;
5471 if (os_sub_overflow(end_addr, va, &cluster_size)) {
5472 cluster_size = 0 - (upl_size_t)PAGE_SIZE;
5473 }
5474 fault_info.cluster_size = cluster_size;
5475
5476 do {
5477 prot = VM_PROT_NONE;
5478
5479 vm_object_lock(object);
5480 vm_object_paging_begin(object);
5481 result_page = VM_PAGE_NULL;
5482 result = vm_fault_page(
5483 object,
5484 (VME_OFFSET(entry) +
5485 (va - entry->vme_start)),
5486 VM_PROT_NONE, TRUE,
5487 FALSE, /* page not looked up */
5488 &prot, &result_page, &top_page,
5489 (int *)0,
5490 NULL, map->no_zero_fill,
5491 FALSE, &fault_info);
5492 } while (result == VM_FAULT_RETRY);
5493
5494 /*
5495 * If this was a mapping to a file on a device that has been forcibly
5496 * unmounted, then we won't get a page back from vm_fault_page(). Just
5497 * move on to the next one in case the remaining pages are mapped from
5498 * different objects. During a forced unmount, the object is terminated
5499 * so the alive flag will be false if this happens. A forced unmount will
5500 * will occur when an external disk is unplugged before the user does an
5501 * eject, so we don't want to panic in that situation.
5502 */
5503
5504 if (result == VM_FAULT_MEMORY_ERROR && !object->alive) {
5505 continue;
5506 }
5507
5508 if (result == VM_FAULT_MEMORY_ERROR &&
5509 object == kernel_object) {
5510 /*
5511 * This must have been allocated with
5512 * KMA_KOBJECT and KMA_VAONLY and there's
5513 * no physical page at this offset.
5514 * We're done (no page to free).
5515 */
5516 assert(deallocate);
5517 continue;
5518 }
5519
5520 if (result != VM_FAULT_SUCCESS) {
5521 panic("vm_fault_unwire: failure");
5522 }
5523
5524 result_object = VM_PAGE_OBJECT(result_page);
5525
5526 if (deallocate) {
5527 assert(VM_PAGE_GET_PHYS_PAGE(result_page) !=
5528 vm_page_fictitious_addr);
5529 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(result_page));
5530 if (VM_PAGE_WIRED(result_page)) {
5531 unwired_pages++;
5532 }
5533 VM_PAGE_FREE(result_page);
5534 } else {
5535 if ((pmap) && (VM_PAGE_GET_PHYS_PAGE(result_page) != vm_page_guard_addr)) {
5536 pmap_change_wiring(pmap,
5537 pmap_addr + (va - entry->vme_start), FALSE);
5538 }
5539
5540
5541 if (VM_PAGE_WIRED(result_page)) {
5542 vm_page_lockspin_queues();
5543 vm_page_unwire(result_page, TRUE);
5544 vm_page_unlock_queues();
5545 unwired_pages++;
5546 }
5547 if (entry->zero_wired_pages) {
5548 pmap_zero_page(VM_PAGE_GET_PHYS_PAGE(result_page));
5549 entry->zero_wired_pages = FALSE;
5550 }
5551
5552 PAGE_WAKEUP_DONE(result_page);
5553 }
5554 vm_fault_cleanup(result_object, top_page);
5555 }
5556 }
5557
5558 /*
5559 * Inform the physical mapping system that the range
5560 * of addresses may fault, so that page tables and
5561 * such may be unwired themselves.
5562 */
5563
5564 pmap_pageable(pmap, pmap_addr,
5565 pmap_addr + (end_addr - entry->vme_start), TRUE);
5566
5567 if (kernel_object == object) {
5568 vm_tag_update_size(fault_info.user_tag, -ptoa_64(unwired_pages));
5569 }
5570 }
5571
5572 /*
5573 * vm_fault_wire_fast:
5574 *
5575 * Handle common case of a wire down page fault at the given address.
5576 * If successful, the page is inserted into the associated physical map.
5577 * The map entry is passed in to avoid the overhead of a map lookup.
5578 *
5579 * NOTE: the given address should be truncated to the
5580 * proper page address.
5581 *
5582 * KERN_SUCCESS is returned if the page fault is handled; otherwise,
5583 * a standard error specifying why the fault is fatal is returned.
5584 *
5585 * The map in question must be referenced, and remains so.
5586 * Caller has a read lock on the map.
5587 *
5588 * This is a stripped version of vm_fault() for wiring pages. Anything
5589 * other than the common case will return KERN_FAILURE, and the caller
5590 * is expected to call vm_fault().
5591 */
5592 static kern_return_t
5593 vm_fault_wire_fast(
5594 __unused vm_map_t map,
5595 vm_map_offset_t va,
5596 __unused vm_prot_t caller_prot,
5597 vm_tag_t wire_tag,
5598 vm_map_entry_t entry,
5599 pmap_t pmap,
5600 vm_map_offset_t pmap_addr,
5601 ppnum_t *physpage_p)
5602 {
5603 vm_object_t object;
5604 vm_object_offset_t offset;
5605 vm_page_t m;
5606 vm_prot_t prot;
5607 thread_t thread = current_thread();
5608 int type_of_fault;
5609 kern_return_t kr;
5610 struct vm_object_fault_info fault_info = {};
5611
5612 VM_STAT_INCR(faults);
5613
5614 if (thread != THREAD_NULL && thread->task != TASK_NULL) {
5615 thread->task->faults++;
5616 }
5617
5618 /*
5619 * Recovery actions
5620 */
5621
5622 #undef RELEASE_PAGE
5623 #define RELEASE_PAGE(m) { \
5624 PAGE_WAKEUP_DONE(m); \
5625 vm_page_lockspin_queues(); \
5626 vm_page_unwire(m, TRUE); \
5627 vm_page_unlock_queues(); \
5628 }
5629
5630
5631 #undef UNLOCK_THINGS
5632 #define UNLOCK_THINGS { \
5633 vm_object_paging_end(object); \
5634 vm_object_unlock(object); \
5635 }
5636
5637 #undef UNLOCK_AND_DEALLOCATE
5638 #define UNLOCK_AND_DEALLOCATE { \
5639 UNLOCK_THINGS; \
5640 vm_object_deallocate(object); \
5641 }
5642 /*
5643 * Give up and have caller do things the hard way.
5644 */
5645
5646 #define GIVE_UP { \
5647 UNLOCK_AND_DEALLOCATE; \
5648 return(KERN_FAILURE); \
5649 }
5650
5651
5652 /*
5653 * If this entry is not directly to a vm_object, bail out.
5654 */
5655 if (entry->is_sub_map) {
5656 assert(physpage_p == NULL);
5657 return KERN_FAILURE;
5658 }
5659
5660 /*
5661 * Find the backing store object and offset into it.
5662 */
5663
5664 object = VME_OBJECT(entry);
5665 offset = (va - entry->vme_start) + VME_OFFSET(entry);
5666 prot = entry->protection;
5667
5668 /*
5669 * Make a reference to this object to prevent its
5670 * disposal while we are messing with it.
5671 */
5672
5673 vm_object_lock(object);
5674 vm_object_reference_locked(object);
5675 vm_object_paging_begin(object);
5676
5677 /*
5678 * INVARIANTS (through entire routine):
5679 *
5680 * 1) At all times, we must either have the object
5681 * lock or a busy page in some object to prevent
5682 * some other thread from trying to bring in
5683 * the same page.
5684 *
5685 * 2) Once we have a busy page, we must remove it from
5686 * the pageout queues, so that the pageout daemon
5687 * will not grab it away.
5688 *
5689 */
5690
5691 /*
5692 * Look for page in top-level object. If it's not there or
5693 * there's something going on, give up.
5694 */
5695 m = vm_page_lookup(object, offset);
5696 if ((m == VM_PAGE_NULL) || (m->vmp_busy) ||
5697 (m->vmp_unusual && (m->vmp_error || m->vmp_restart || m->vmp_absent))) {
5698 GIVE_UP;
5699 }
5700 if (m->vmp_fictitious &&
5701 VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
5702 /*
5703 * Guard pages are fictitious pages and are never
5704 * entered into a pmap, so let's say it's been wired...
5705 */
5706 kr = KERN_SUCCESS;
5707 goto done;
5708 }
5709
5710 /*
5711 * Wire the page down now. All bail outs beyond this
5712 * point must unwire the page.
5713 */
5714
5715 vm_page_lockspin_queues();
5716 vm_page_wire(m, wire_tag, TRUE);
5717 vm_page_unlock_queues();
5718
5719 /*
5720 * Mark page busy for other threads.
5721 */
5722 assert(!m->vmp_busy);
5723 m->vmp_busy = TRUE;
5724 assert(!m->vmp_absent);
5725
5726 /*
5727 * Give up if the page is being written and there's a copy object
5728 */
5729 if ((object->copy != VM_OBJECT_NULL) && (prot & VM_PROT_WRITE)) {
5730 RELEASE_PAGE(m);
5731 GIVE_UP;
5732 }
5733
5734 fault_info.user_tag = VME_ALIAS(entry);
5735 fault_info.pmap_options = 0;
5736 if (entry->iokit_acct ||
5737 (!entry->is_sub_map && !entry->use_pmap)) {
5738 fault_info.pmap_options |= PMAP_OPTIONS_ALT_ACCT;
5739 }
5740
5741 /*
5742 * Put this page into the physical map.
5743 */
5744 type_of_fault = DBG_CACHE_HIT_FAULT;
5745 kr = vm_fault_enter(m,
5746 pmap,
5747 pmap_addr,
5748 prot,
5749 prot,
5750 TRUE, /* wired */
5751 FALSE, /* change_wiring */
5752 wire_tag,
5753 &fault_info,
5754 NULL,
5755 &type_of_fault);
5756 if (kr != KERN_SUCCESS) {
5757 RELEASE_PAGE(m);
5758 GIVE_UP;
5759 }
5760
5761 done:
5762 /*
5763 * Unlock everything, and return
5764 */
5765
5766 if (physpage_p) {
5767 /* for vm_map_wire_and_extract() */
5768 if (kr == KERN_SUCCESS) {
5769 assert(object == VM_PAGE_OBJECT(m));
5770 *physpage_p = VM_PAGE_GET_PHYS_PAGE(m);
5771 if (prot & VM_PROT_WRITE) {
5772 vm_object_lock_assert_exclusive(object);
5773 m->vmp_dirty = TRUE;
5774 }
5775 } else {
5776 *physpage_p = 0;
5777 }
5778 }
5779
5780 PAGE_WAKEUP_DONE(m);
5781 UNLOCK_AND_DEALLOCATE;
5782
5783 return kr;
5784 }
5785
5786 /*
5787 * Routine: vm_fault_copy_cleanup
5788 * Purpose:
5789 * Release a page used by vm_fault_copy.
5790 */
5791
5792 static void
5793 vm_fault_copy_cleanup(
5794 vm_page_t page,
5795 vm_page_t top_page)
5796 {
5797 vm_object_t object = VM_PAGE_OBJECT(page);
5798
5799 vm_object_lock(object);
5800 PAGE_WAKEUP_DONE(page);
5801 if (!VM_PAGE_PAGEABLE(page)) {
5802 vm_page_lockspin_queues();
5803 if (!VM_PAGE_PAGEABLE(page)) {
5804 vm_page_activate(page);
5805 }
5806 vm_page_unlock_queues();
5807 }
5808 vm_fault_cleanup(object, top_page);
5809 }
5810
5811 static void
5812 vm_fault_copy_dst_cleanup(
5813 vm_page_t page)
5814 {
5815 vm_object_t object;
5816
5817 if (page != VM_PAGE_NULL) {
5818 object = VM_PAGE_OBJECT(page);
5819 vm_object_lock(object);
5820 vm_page_lockspin_queues();
5821 vm_page_unwire(page, TRUE);
5822 vm_page_unlock_queues();
5823 vm_object_paging_end(object);
5824 vm_object_unlock(object);
5825 }
5826 }
5827
5828 /*
5829 * Routine: vm_fault_copy
5830 *
5831 * Purpose:
5832 * Copy pages from one virtual memory object to another --
5833 * neither the source nor destination pages need be resident.
5834 *
5835 * Before actually copying a page, the version associated with
5836 * the destination address map wil be verified.
5837 *
5838 * In/out conditions:
5839 * The caller must hold a reference, but not a lock, to
5840 * each of the source and destination objects and to the
5841 * destination map.
5842 *
5843 * Results:
5844 * Returns KERN_SUCCESS if no errors were encountered in
5845 * reading or writing the data. Returns KERN_INTERRUPTED if
5846 * the operation was interrupted (only possible if the
5847 * "interruptible" argument is asserted). Other return values
5848 * indicate a permanent error in copying the data.
5849 *
5850 * The actual amount of data copied will be returned in the
5851 * "copy_size" argument. In the event that the destination map
5852 * verification failed, this amount may be less than the amount
5853 * requested.
5854 */
5855 kern_return_t
5856 vm_fault_copy(
5857 vm_object_t src_object,
5858 vm_object_offset_t src_offset,
5859 vm_map_size_t *copy_size, /* INOUT */
5860 vm_object_t dst_object,
5861 vm_object_offset_t dst_offset,
5862 vm_map_t dst_map,
5863 vm_map_version_t *dst_version,
5864 int interruptible)
5865 {
5866 vm_page_t result_page;
5867
5868 vm_page_t src_page;
5869 vm_page_t src_top_page;
5870 vm_prot_t src_prot;
5871
5872 vm_page_t dst_page;
5873 vm_page_t dst_top_page;
5874 vm_prot_t dst_prot;
5875
5876 vm_map_size_t amount_left;
5877 vm_object_t old_copy_object;
5878 vm_object_t result_page_object = NULL;
5879 kern_return_t error = 0;
5880 vm_fault_return_t result;
5881
5882 vm_map_size_t part_size;
5883 struct vm_object_fault_info fault_info_src = {};
5884 struct vm_object_fault_info fault_info_dst = {};
5885
5886 /*
5887 * In order not to confuse the clustered pageins, align
5888 * the different offsets on a page boundary.
5889 */
5890
5891 #define RETURN(x) \
5892 MACRO_BEGIN \
5893 *copy_size -= amount_left; \
5894 MACRO_RETURN(x); \
5895 MACRO_END
5896
5897 amount_left = *copy_size;
5898
5899 fault_info_src.interruptible = interruptible;
5900 fault_info_src.behavior = VM_BEHAVIOR_SEQUENTIAL;
5901 fault_info_src.lo_offset = vm_object_trunc_page(src_offset);
5902 fault_info_src.hi_offset = fault_info_src.lo_offset + amount_left;
5903 fault_info_src.stealth = TRUE;
5904
5905 fault_info_dst.interruptible = interruptible;
5906 fault_info_dst.behavior = VM_BEHAVIOR_SEQUENTIAL;
5907 fault_info_dst.lo_offset = vm_object_trunc_page(dst_offset);
5908 fault_info_dst.hi_offset = fault_info_dst.lo_offset + amount_left;
5909 fault_info_dst.stealth = TRUE;
5910
5911 do { /* while (amount_left > 0) */
5912 /*
5913 * There may be a deadlock if both source and destination
5914 * pages are the same. To avoid this deadlock, the copy must
5915 * start by getting the destination page in order to apply
5916 * COW semantics if any.
5917 */
5918
5919 RetryDestinationFault:;
5920
5921 dst_prot = VM_PROT_WRITE | VM_PROT_READ;
5922
5923 vm_object_lock(dst_object);
5924 vm_object_paging_begin(dst_object);
5925
5926 /* cap cluster size at maximum UPL size */
5927 upl_size_t cluster_size;
5928 if (os_convert_overflow(amount_left, &cluster_size)) {
5929 cluster_size = 0 - (upl_size_t)PAGE_SIZE;
5930 }
5931 fault_info_dst.cluster_size = cluster_size;
5932
5933 dst_page = VM_PAGE_NULL;
5934 result = vm_fault_page(dst_object,
5935 vm_object_trunc_page(dst_offset),
5936 VM_PROT_WRITE | VM_PROT_READ,
5937 FALSE,
5938 FALSE, /* page not looked up */
5939 &dst_prot, &dst_page, &dst_top_page,
5940 (int *)0,
5941 &error,
5942 dst_map->no_zero_fill,
5943 FALSE, &fault_info_dst);
5944 switch (result) {
5945 case VM_FAULT_SUCCESS:
5946 break;
5947 case VM_FAULT_RETRY:
5948 goto RetryDestinationFault;
5949 case VM_FAULT_MEMORY_SHORTAGE:
5950 if (vm_page_wait(interruptible)) {
5951 goto RetryDestinationFault;
5952 }
5953 /* fall thru */
5954 case VM_FAULT_INTERRUPTED:
5955 RETURN(MACH_SEND_INTERRUPTED);
5956 case VM_FAULT_SUCCESS_NO_VM_PAGE:
5957 /* success but no VM page: fail the copy */
5958 vm_object_paging_end(dst_object);
5959 vm_object_unlock(dst_object);
5960 /*FALLTHROUGH*/
5961 case VM_FAULT_MEMORY_ERROR:
5962 if (error) {
5963 return error;
5964 } else {
5965 return KERN_MEMORY_ERROR;
5966 }
5967 default:
5968 panic("vm_fault_copy: unexpected error 0x%x from "
5969 "vm_fault_page()\n", result);
5970 }
5971 assert((dst_prot & VM_PROT_WRITE) != VM_PROT_NONE);
5972
5973 assert(dst_object == VM_PAGE_OBJECT(dst_page));
5974 old_copy_object = dst_object->copy;
5975
5976 /*
5977 * There exists the possiblity that the source and
5978 * destination page are the same. But we can't
5979 * easily determine that now. If they are the
5980 * same, the call to vm_fault_page() for the
5981 * destination page will deadlock. To prevent this we
5982 * wire the page so we can drop busy without having
5983 * the page daemon steal the page. We clean up the
5984 * top page but keep the paging reference on the object
5985 * holding the dest page so it doesn't go away.
5986 */
5987
5988 vm_page_lockspin_queues();
5989 vm_page_wire(dst_page, VM_KERN_MEMORY_OSFMK, TRUE);
5990 vm_page_unlock_queues();
5991 PAGE_WAKEUP_DONE(dst_page);
5992 vm_object_unlock(dst_object);
5993
5994 if (dst_top_page != VM_PAGE_NULL) {
5995 vm_object_lock(dst_object);
5996 VM_PAGE_FREE(dst_top_page);
5997 vm_object_paging_end(dst_object);
5998 vm_object_unlock(dst_object);
5999 }
6000
6001 RetrySourceFault:;
6002
6003 if (src_object == VM_OBJECT_NULL) {
6004 /*
6005 * No source object. We will just
6006 * zero-fill the page in dst_object.
6007 */
6008 src_page = VM_PAGE_NULL;
6009 result_page = VM_PAGE_NULL;
6010 } else {
6011 vm_object_lock(src_object);
6012 src_page = vm_page_lookup(src_object,
6013 vm_object_trunc_page(src_offset));
6014 if (src_page == dst_page) {
6015 src_prot = dst_prot;
6016 result_page = VM_PAGE_NULL;
6017 } else {
6018 src_prot = VM_PROT_READ;
6019 vm_object_paging_begin(src_object);
6020
6021 /* cap cluster size at maximum UPL size */
6022 if (os_convert_overflow(amount_left, &cluster_size)) {
6023 cluster_size = 0 - (upl_size_t)PAGE_SIZE;
6024 }
6025 fault_info_src.cluster_size = cluster_size;
6026
6027 result_page = VM_PAGE_NULL;
6028 result = vm_fault_page(
6029 src_object,
6030 vm_object_trunc_page(src_offset),
6031 VM_PROT_READ, FALSE,
6032 FALSE, /* page not looked up */
6033 &src_prot,
6034 &result_page, &src_top_page,
6035 (int *)0, &error, FALSE,
6036 FALSE, &fault_info_src);
6037
6038 switch (result) {
6039 case VM_FAULT_SUCCESS:
6040 break;
6041 case VM_FAULT_RETRY:
6042 goto RetrySourceFault;
6043 case VM_FAULT_MEMORY_SHORTAGE:
6044 if (vm_page_wait(interruptible)) {
6045 goto RetrySourceFault;
6046 }
6047 /* fall thru */
6048 case VM_FAULT_INTERRUPTED:
6049 vm_fault_copy_dst_cleanup(dst_page);
6050 RETURN(MACH_SEND_INTERRUPTED);
6051 case VM_FAULT_SUCCESS_NO_VM_PAGE:
6052 /* success but no VM page: fail */
6053 vm_object_paging_end(src_object);
6054 vm_object_unlock(src_object);
6055 /*FALLTHROUGH*/
6056 case VM_FAULT_MEMORY_ERROR:
6057 vm_fault_copy_dst_cleanup(dst_page);
6058 if (error) {
6059 return error;
6060 } else {
6061 return KERN_MEMORY_ERROR;
6062 }
6063 default:
6064 panic("vm_fault_copy(2): unexpected "
6065 "error 0x%x from "
6066 "vm_fault_page()\n", result);
6067 }
6068
6069 result_page_object = VM_PAGE_OBJECT(result_page);
6070 assert((src_top_page == VM_PAGE_NULL) ==
6071 (result_page_object == src_object));
6072 }
6073 assert((src_prot & VM_PROT_READ) != VM_PROT_NONE);
6074 vm_object_unlock(result_page_object);
6075 }
6076
6077 vm_map_lock_read(dst_map);
6078
6079 if (!vm_map_verify(dst_map, dst_version)) {
6080 vm_map_unlock_read(dst_map);
6081 if (result_page != VM_PAGE_NULL && src_page != dst_page) {
6082 vm_fault_copy_cleanup(result_page, src_top_page);
6083 }
6084 vm_fault_copy_dst_cleanup(dst_page);
6085 break;
6086 }
6087 assert(dst_object == VM_PAGE_OBJECT(dst_page));
6088
6089 vm_object_lock(dst_object);
6090
6091 if (dst_object->copy != old_copy_object) {
6092 vm_object_unlock(dst_object);
6093 vm_map_unlock_read(dst_map);
6094 if (result_page != VM_PAGE_NULL && src_page != dst_page) {
6095 vm_fault_copy_cleanup(result_page, src_top_page);
6096 }
6097 vm_fault_copy_dst_cleanup(dst_page);
6098 break;
6099 }
6100 vm_object_unlock(dst_object);
6101
6102 /*
6103 * Copy the page, and note that it is dirty
6104 * immediately.
6105 */
6106
6107 if (!page_aligned(src_offset) ||
6108 !page_aligned(dst_offset) ||
6109 !page_aligned(amount_left)) {
6110 vm_object_offset_t src_po,
6111 dst_po;
6112
6113 src_po = src_offset - vm_object_trunc_page(src_offset);
6114 dst_po = dst_offset - vm_object_trunc_page(dst_offset);
6115
6116 if (dst_po > src_po) {
6117 part_size = PAGE_SIZE - dst_po;
6118 } else {
6119 part_size = PAGE_SIZE - src_po;
6120 }
6121 if (part_size > (amount_left)) {
6122 part_size = amount_left;
6123 }
6124
6125 if (result_page == VM_PAGE_NULL) {
6126 assert((vm_offset_t) dst_po == dst_po);
6127 assert((vm_size_t) part_size == part_size);
6128 vm_page_part_zero_fill(dst_page,
6129 (vm_offset_t) dst_po,
6130 (vm_size_t) part_size);
6131 } else {
6132 assert((vm_offset_t) src_po == src_po);
6133 assert((vm_offset_t) dst_po == dst_po);
6134 assert((vm_size_t) part_size == part_size);
6135 vm_page_part_copy(result_page,
6136 (vm_offset_t) src_po,
6137 dst_page,
6138 (vm_offset_t) dst_po,
6139 (vm_size_t)part_size);
6140 if (!dst_page->vmp_dirty) {
6141 vm_object_lock(dst_object);
6142 SET_PAGE_DIRTY(dst_page, TRUE);
6143 vm_object_unlock(dst_object);
6144 }
6145 }
6146 } else {
6147 part_size = PAGE_SIZE;
6148
6149 if (result_page == VM_PAGE_NULL) {
6150 vm_page_zero_fill(dst_page);
6151 } else {
6152 vm_object_lock(result_page_object);
6153 vm_page_copy(result_page, dst_page);
6154 vm_object_unlock(result_page_object);
6155
6156 if (!dst_page->vmp_dirty) {
6157 vm_object_lock(dst_object);
6158 SET_PAGE_DIRTY(dst_page, TRUE);
6159 vm_object_unlock(dst_object);
6160 }
6161 }
6162 }
6163
6164 /*
6165 * Unlock everything, and return
6166 */
6167
6168 vm_map_unlock_read(dst_map);
6169
6170 if (result_page != VM_PAGE_NULL && src_page != dst_page) {
6171 vm_fault_copy_cleanup(result_page, src_top_page);
6172 }
6173 vm_fault_copy_dst_cleanup(dst_page);
6174
6175 amount_left -= part_size;
6176 src_offset += part_size;
6177 dst_offset += part_size;
6178 } while (amount_left > 0);
6179
6180 RETURN(KERN_SUCCESS);
6181 #undef RETURN
6182
6183 /*NOTREACHED*/
6184 }
6185
6186 #if VM_FAULT_CLASSIFY
6187 /*
6188 * Temporary statistics gathering support.
6189 */
6190
6191 /*
6192 * Statistics arrays:
6193 */
6194 #define VM_FAULT_TYPES_MAX 5
6195 #define VM_FAULT_LEVEL_MAX 8
6196
6197 int vm_fault_stats[VM_FAULT_TYPES_MAX][VM_FAULT_LEVEL_MAX];
6198
6199 #define VM_FAULT_TYPE_ZERO_FILL 0
6200 #define VM_FAULT_TYPE_MAP_IN 1
6201 #define VM_FAULT_TYPE_PAGER 2
6202 #define VM_FAULT_TYPE_COPY 3
6203 #define VM_FAULT_TYPE_OTHER 4
6204
6205
6206 void
6207 vm_fault_classify(vm_object_t object,
6208 vm_object_offset_t offset,
6209 vm_prot_t fault_type)
6210 {
6211 int type, level = 0;
6212 vm_page_t m;
6213
6214 while (TRUE) {
6215 m = vm_page_lookup(object, offset);
6216 if (m != VM_PAGE_NULL) {
6217 if (m->vmp_busy || m->vmp_error || m->vmp_restart || m->vmp_absent) {
6218 type = VM_FAULT_TYPE_OTHER;
6219 break;
6220 }
6221 if (((fault_type & VM_PROT_WRITE) == 0) ||
6222 ((level == 0) && object->copy == VM_OBJECT_NULL)) {
6223 type = VM_FAULT_TYPE_MAP_IN;
6224 break;
6225 }
6226 type = VM_FAULT_TYPE_COPY;
6227 break;
6228 } else {
6229 if (object->pager_created) {
6230 type = VM_FAULT_TYPE_PAGER;
6231 break;
6232 }
6233 if (object->shadow == VM_OBJECT_NULL) {
6234 type = VM_FAULT_TYPE_ZERO_FILL;
6235 break;
6236 }
6237
6238 offset += object->vo_shadow_offset;
6239 object = object->shadow;
6240 level++;
6241 continue;
6242 }
6243 }
6244
6245 if (level > VM_FAULT_LEVEL_MAX) {
6246 level = VM_FAULT_LEVEL_MAX;
6247 }
6248
6249 vm_fault_stats[type][level] += 1;
6250
6251 return;
6252 }
6253
6254 /* cleanup routine to call from debugger */
6255
6256 void
6257 vm_fault_classify_init(void)
6258 {
6259 int type, level;
6260
6261 for (type = 0; type < VM_FAULT_TYPES_MAX; type++) {
6262 for (level = 0; level < VM_FAULT_LEVEL_MAX; level++) {
6263 vm_fault_stats[type][level] = 0;
6264 }
6265 }
6266
6267 return;
6268 }
6269 #endif /* VM_FAULT_CLASSIFY */
6270
6271 vm_offset_t
6272 kdp_lightweight_fault(vm_map_t map, vm_offset_t cur_target_addr)
6273 {
6274 vm_map_entry_t entry;
6275 vm_object_t object;
6276 vm_offset_t object_offset;
6277 vm_page_t m;
6278 int compressor_external_state, compressed_count_delta;
6279 int compressor_flags = (C_DONT_BLOCK | C_KEEP | C_KDP);
6280 int my_fault_type = VM_PROT_READ;
6281 kern_return_t kr;
6282
6283 if (not_in_kdp) {
6284 panic("kdp_lightweight_fault called from outside of debugger context");
6285 }
6286
6287 assert(map != VM_MAP_NULL);
6288
6289 assert((cur_target_addr & PAGE_MASK) == 0);
6290 if ((cur_target_addr & PAGE_MASK) != 0) {
6291 return 0;
6292 }
6293
6294 if (kdp_lck_rw_lock_is_acquired_exclusive(&map->lock)) {
6295 return 0;
6296 }
6297
6298 if (!vm_map_lookup_entry(map, cur_target_addr, &entry)) {
6299 return 0;
6300 }
6301
6302 if (entry->is_sub_map) {
6303 return 0;
6304 }
6305
6306 object = VME_OBJECT(entry);
6307 if (object == VM_OBJECT_NULL) {
6308 return 0;
6309 }
6310
6311 object_offset = cur_target_addr - entry->vme_start + VME_OFFSET(entry);
6312
6313 while (TRUE) {
6314 if (kdp_lck_rw_lock_is_acquired_exclusive(&object->Lock)) {
6315 return 0;
6316 }
6317
6318 if (object->pager_created && (object->paging_in_progress ||
6319 object->activity_in_progress)) {
6320 return 0;
6321 }
6322
6323 m = kdp_vm_page_lookup(object, object_offset);
6324
6325 if (m != VM_PAGE_NULL) {
6326 if ((object->wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
6327 return 0;
6328 }
6329
6330 if (m->vmp_laundry || m->vmp_busy || m->vmp_free_when_done || m->vmp_absent || m->vmp_error || m->vmp_cleaning ||
6331 m->vmp_overwriting || m->vmp_restart || m->vmp_unusual) {
6332 return 0;
6333 }
6334
6335 assert(!m->vmp_private);
6336 if (m->vmp_private) {
6337 return 0;
6338 }
6339
6340 assert(!m->vmp_fictitious);
6341 if (m->vmp_fictitious) {
6342 return 0;
6343 }
6344
6345 assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
6346 if (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
6347 return 0;
6348 }
6349
6350 return ptoa(VM_PAGE_GET_PHYS_PAGE(m));
6351 }
6352
6353 compressor_external_state = VM_EXTERNAL_STATE_UNKNOWN;
6354
6355 if (object->pager_created && MUST_ASK_PAGER(object, object_offset, compressor_external_state)) {
6356 if (compressor_external_state == VM_EXTERNAL_STATE_EXISTS) {
6357 kr = vm_compressor_pager_get(object->pager, (object_offset + object->paging_offset),
6358 kdp_compressor_decompressed_page_ppnum, &my_fault_type,
6359 compressor_flags, &compressed_count_delta);
6360 if (kr == KERN_SUCCESS) {
6361 return kdp_compressor_decompressed_page_paddr;
6362 } else {
6363 return 0;
6364 }
6365 }
6366 }
6367
6368 if (object->shadow == VM_OBJECT_NULL) {
6369 return 0;
6370 }
6371
6372 object_offset += object->vo_shadow_offset;
6373 object = object->shadow;
6374 }
6375 }
6376
6377 /*
6378 * vm_page_validate_cs_fast():
6379 * Performs a few quick checks to determine if the page's code signature
6380 * really needs to be fully validated. It could:
6381 * 1. have been modified (i.e. automatically tainted),
6382 * 2. have already been validated,
6383 * 3. have already been found to be tainted,
6384 * 4. no longer have a backing store.
6385 * Returns FALSE if the page needs to be fully validated.
6386 */
6387 static boolean_t
6388 vm_page_validate_cs_fast(
6389 vm_page_t page)
6390 {
6391 vm_object_t object;
6392
6393 object = VM_PAGE_OBJECT(page);
6394 vm_object_lock_assert_held(object);
6395
6396 if (page->vmp_wpmapped && !page->vmp_cs_tainted) {
6397 /*
6398 * This page was mapped for "write" access sometime in the
6399 * past and could still be modifiable in the future.
6400 * Consider it tainted.
6401 * [ If the page was already found to be "tainted", no
6402 * need to re-validate. ]
6403 */
6404 vm_object_lock_assert_exclusive(object);
6405 page->vmp_cs_validated = TRUE;
6406 page->vmp_cs_tainted = TRUE;
6407 if (cs_debug) {
6408 printf("CODESIGNING: %s: "
6409 "page %p obj %p off 0x%llx "
6410 "was modified\n",
6411 __FUNCTION__,
6412 page, object, page->vmp_offset);
6413 }
6414 vm_cs_validated_dirtied++;
6415 }
6416
6417 if (page->vmp_cs_validated || page->vmp_cs_tainted) {
6418 return TRUE;
6419 }
6420 vm_object_lock_assert_exclusive(object);
6421
6422 #if CHECK_CS_VALIDATION_BITMAP
6423 kern_return_t kr;
6424
6425 kr = vnode_pager_cs_check_validation_bitmap(
6426 object->pager,
6427 page->vmp_offset + object->paging_offset,
6428 CS_BITMAP_CHECK);
6429 if (kr == KERN_SUCCESS) {
6430 page->vmp_cs_validated = TRUE;
6431 page->vmp_cs_tainted = FALSE;
6432 vm_cs_bitmap_validated++;
6433 return TRUE;
6434 }
6435 #endif /* CHECK_CS_VALIDATION_BITMAP */
6436
6437 if (!object->alive || object->terminating || object->pager == NULL) {
6438 /*
6439 * The object is terminating and we don't have its pager
6440 * so we can't validate the data...
6441 */
6442 return TRUE;
6443 }
6444
6445 /* we need to really validate this page */
6446 vm_object_lock_assert_exclusive(object);
6447 return FALSE;
6448 }
6449
6450 void
6451 vm_page_validate_cs_mapped_slow(
6452 vm_page_t page,
6453 const void *kaddr)
6454 {
6455 vm_object_t object;
6456 memory_object_offset_t mo_offset;
6457 memory_object_t pager;
6458 struct vnode *vnode;
6459 boolean_t validated;
6460 unsigned tainted;
6461
6462 assert(page->vmp_busy);
6463 object = VM_PAGE_OBJECT(page);
6464 vm_object_lock_assert_exclusive(object);
6465
6466 vm_cs_validates++;
6467
6468 /*
6469 * Since we get here to validate a page that was brought in by
6470 * the pager, we know that this pager is all setup and ready
6471 * by now.
6472 */
6473 assert(object->code_signed);
6474 assert(!object->internal);
6475 assert(object->pager != NULL);
6476 assert(object->pager_ready);
6477
6478 pager = object->pager;
6479 assert(object->paging_in_progress);
6480 vnode = vnode_pager_lookup_vnode(pager);
6481 mo_offset = page->vmp_offset + object->paging_offset;
6482
6483 /* verify the SHA1 hash for this page */
6484 tainted = 0;
6485 validated = cs_validate_range(vnode,
6486 pager,
6487 mo_offset,
6488 (const void *)((const char *)kaddr),
6489 PAGE_SIZE_64,
6490 &tainted);
6491
6492 if (tainted & CS_VALIDATE_TAINTED) {
6493 page->vmp_cs_tainted = TRUE;
6494 }
6495 if (tainted & CS_VALIDATE_NX) {
6496 page->vmp_cs_nx = TRUE;
6497 }
6498 if (validated) {
6499 page->vmp_cs_validated = TRUE;
6500 }
6501
6502 #if CHECK_CS_VALIDATION_BITMAP
6503 if (page->vmp_cs_validated && !page->vmp_cs_tainted) {
6504 vnode_pager_cs_check_validation_bitmap(object->pager,
6505 mo_offset,
6506 CS_BITMAP_SET);
6507 }
6508 #endif /* CHECK_CS_VALIDATION_BITMAP */
6509 }
6510
6511 void
6512 vm_page_validate_cs_mapped(
6513 vm_page_t page,
6514 const void *kaddr)
6515 {
6516 if (!vm_page_validate_cs_fast(page)) {
6517 vm_page_validate_cs_mapped_slow(page, kaddr);
6518 }
6519 }
6520
6521 void
6522 vm_page_validate_cs(
6523 vm_page_t page)
6524 {
6525 vm_object_t object;
6526 vm_object_offset_t offset;
6527 vm_map_offset_t koffset;
6528 vm_map_size_t ksize;
6529 vm_offset_t kaddr;
6530 kern_return_t kr;
6531 boolean_t busy_page;
6532 boolean_t need_unmap;
6533
6534 object = VM_PAGE_OBJECT(page);
6535 vm_object_lock_assert_held(object);
6536
6537 if (vm_page_validate_cs_fast(page)) {
6538 return;
6539 }
6540 vm_object_lock_assert_exclusive(object);
6541
6542 assert(object->code_signed);
6543 offset = page->vmp_offset;
6544
6545 busy_page = page->vmp_busy;
6546 if (!busy_page) {
6547 /* keep page busy while we map (and unlock) the VM object */
6548 page->vmp_busy = TRUE;
6549 }
6550
6551 /*
6552 * Take a paging reference on the VM object
6553 * to protect it from collapse or bypass,
6554 * and keep it from disappearing too.
6555 */
6556 vm_object_paging_begin(object);
6557
6558 /* map the page in the kernel address space */
6559 ksize = PAGE_SIZE_64;
6560 koffset = 0;
6561 need_unmap = FALSE;
6562 kr = vm_paging_map_object(page,
6563 object,
6564 offset,
6565 VM_PROT_READ,
6566 FALSE, /* can't unlock object ! */
6567 &ksize,
6568 &koffset,
6569 &need_unmap);
6570 if (kr != KERN_SUCCESS) {
6571 panic("%s: could not map page: 0x%x\n", __FUNCTION__, kr);
6572 }
6573 kaddr = CAST_DOWN(vm_offset_t, koffset);
6574
6575 /* validate the mapped page */
6576 vm_page_validate_cs_mapped_slow(page, (const void *) kaddr);
6577
6578 assert(page->vmp_busy);
6579 assert(object == VM_PAGE_OBJECT(page));
6580 vm_object_lock_assert_exclusive(object);
6581
6582 if (!busy_page) {
6583 PAGE_WAKEUP_DONE(page);
6584 }
6585 if (need_unmap) {
6586 /* unmap the map from the kernel address space */
6587 vm_paging_unmap_object(object, koffset, koffset + ksize);
6588 koffset = 0;
6589 ksize = 0;
6590 kaddr = 0;
6591 }
6592 vm_object_paging_end(object);
6593 }
6594
6595 void
6596 vm_page_validate_cs_mapped_chunk(
6597 vm_page_t page,
6598 const void *kaddr,
6599 vm_offset_t chunk_offset,
6600 vm_size_t chunk_size,
6601 boolean_t *validated_p,
6602 unsigned *tainted_p)
6603 {
6604 vm_object_t object;
6605 vm_object_offset_t offset, offset_in_page;
6606 memory_object_t pager;
6607 struct vnode *vnode;
6608 boolean_t validated;
6609 unsigned tainted;
6610
6611 *validated_p = FALSE;
6612 *tainted_p = 0;
6613
6614 assert(page->vmp_busy);
6615 object = VM_PAGE_OBJECT(page);
6616 vm_object_lock_assert_exclusive(object);
6617
6618 assert(object->code_signed);
6619 offset = page->vmp_offset;
6620
6621 if (!object->alive || object->terminating || object->pager == NULL) {
6622 /*
6623 * The object is terminating and we don't have its pager
6624 * so we can't validate the data...
6625 */
6626 return;
6627 }
6628 /*
6629 * Since we get here to validate a page that was brought in by
6630 * the pager, we know that this pager is all setup and ready
6631 * by now.
6632 */
6633 assert(!object->internal);
6634 assert(object->pager != NULL);
6635 assert(object->pager_ready);
6636
6637 pager = object->pager;
6638 assert(object->paging_in_progress);
6639 vnode = vnode_pager_lookup_vnode(pager);
6640
6641 /* verify the signature for this chunk */
6642 offset_in_page = chunk_offset;
6643 assert(offset_in_page < PAGE_SIZE);
6644
6645 tainted = 0;
6646 validated = cs_validate_range(vnode,
6647 pager,
6648 (object->paging_offset +
6649 offset +
6650 offset_in_page),
6651 (const void *)((const char *)kaddr
6652 + offset_in_page),
6653 chunk_size,
6654 &tainted);
6655 if (validated) {
6656 *validated_p = TRUE;
6657 }
6658 if (tainted) {
6659 *tainted_p = tainted;
6660 }
6661 }
6662
6663 static void
6664 vm_rtfrecord_lock(void)
6665 {
6666 lck_spin_lock(&vm_rtfr_slock);
6667 }
6668
6669 static void
6670 vm_rtfrecord_unlock(void)
6671 {
6672 lck_spin_unlock(&vm_rtfr_slock);
6673 }
6674
6675 unsigned int
6676 vmrtfaultinfo_bufsz(void)
6677 {
6678 return vmrtf_num_records * sizeof(vm_rtfault_record_t);
6679 }
6680
6681 #include <kern/backtrace.h>
6682
6683 static void
6684 vm_record_rtfault(thread_t cthread, uint64_t fstart, vm_map_offset_t fault_vaddr, int type_of_fault)
6685 {
6686 uint64_t fend = mach_continuous_time();
6687
6688 uint64_t cfpc = 0;
6689 uint64_t ctid = cthread->thread_id;
6690 uint64_t cupid = get_current_unique_pid();
6691
6692 uintptr_t bpc = 0;
6693 int btr = 0;
6694 bool u64 = false;
6695
6696 /* Capture a single-frame backtrace; this extracts just the program
6697 * counter at the point of the fault into "bpc", and should perform no
6698 * further user stack traversals, thus avoiding copyin()s and further
6699 * faults.
6700 */
6701 unsigned int bfrs = backtrace_thread_user(cthread, &bpc, 1U, &btr, &u64, NULL);
6702
6703 if ((btr == 0) && (bfrs > 0)) {
6704 cfpc = bpc;
6705 }
6706
6707 assert((fstart != 0) && fend >= fstart);
6708 vm_rtfrecord_lock();
6709 assert(vmrtfrs.vmrtfr_curi <= vmrtfrs.vmrtfr_maxi);
6710
6711 vmrtfrs.vmrtf_total++;
6712 vm_rtfault_record_t *cvmr = &vmrtfrs.vm_rtf_records[vmrtfrs.vmrtfr_curi++];
6713
6714 cvmr->rtfabstime = fstart;
6715 cvmr->rtfduration = fend - fstart;
6716 cvmr->rtfaddr = fault_vaddr;
6717 cvmr->rtfpc = cfpc;
6718 cvmr->rtftype = type_of_fault;
6719 cvmr->rtfupid = cupid;
6720 cvmr->rtftid = ctid;
6721
6722 if (vmrtfrs.vmrtfr_curi > vmrtfrs.vmrtfr_maxi) {
6723 vmrtfrs.vmrtfr_curi = 0;
6724 }
6725
6726 vm_rtfrecord_unlock();
6727 }
6728
6729 int
6730 vmrtf_extract(uint64_t cupid, __unused boolean_t isroot, int vrecordsz, void *vrecords, int *vmrtfrv)
6731 {
6732 vm_rtfault_record_t *cvmrd = vrecords;
6733 size_t residue = vrecordsz;
6734 int numextracted = 0;
6735 boolean_t early_exit = FALSE;
6736
6737 vm_rtfrecord_lock();
6738
6739 for (int vmfi = 0; vmfi <= vmrtfrs.vmrtfr_maxi; vmfi++) {
6740 if (residue < sizeof(vm_rtfault_record_t)) {
6741 early_exit = TRUE;
6742 break;
6743 }
6744
6745 if (vmrtfrs.vm_rtf_records[vmfi].rtfupid != cupid) {
6746 #if DEVELOPMENT || DEBUG
6747 if (isroot == FALSE) {
6748 continue;
6749 }
6750 #else
6751 continue;
6752 #endif /* DEVDEBUG */
6753 }
6754
6755 *cvmrd = vmrtfrs.vm_rtf_records[vmfi];
6756 cvmrd++;
6757 residue -= sizeof(vm_rtfault_record_t);
6758 numextracted++;
6759 }
6760
6761 vm_rtfrecord_unlock();
6762
6763 *vmrtfrv = numextracted;
6764 return early_exit;
6765 }
mirror of XNU