]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_fault.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4903.231.4.tar.gz
[apple/xnu.git] / osfmk / vm / vm_fault.c
1 /*
2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm_fault.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * Page fault handling module.
63 */
64
65 #include <mach_cluster_stats.h>
66 #include <mach_pagemap.h>
67 #include <libkern/OSAtomic.h>
68
69 #include <mach/mach_types.h>
70 #include <mach/kern_return.h>
71 #include <mach/message.h> /* for error codes */
72 #include <mach/vm_param.h>
73 #include <mach/vm_behavior.h>
74 #include <mach/memory_object.h>
75 /* For memory_object_data_{request,unlock} */
76 #include <mach/sdt.h>
77
78 #include <kern/kern_types.h>
79 #include <kern/host_statistics.h>
80 #include <kern/counters.h>
81 #include <kern/task.h>
82 #include <kern/thread.h>
83 #include <kern/sched_prim.h>
84 #include <kern/host.h>
85 #include <kern/xpr.h>
86 #include <kern/mach_param.h>
87 #include <kern/macro_help.h>
88 #include <kern/zalloc.h>
89 #include <kern/misc_protos.h>
90 #include <kern/policy_internal.h>
91
92 #include <vm/vm_compressor.h>
93 #include <vm/vm_compressor_pager.h>
94 #include <vm/vm_fault.h>
95 #include <vm/vm_map.h>
96 #include <vm/vm_object.h>
97 #include <vm/vm_page.h>
98 #include <vm/vm_kern.h>
99 #include <vm/pmap.h>
100 #include <vm/vm_pageout.h>
101 #include <vm/vm_protos.h>
102 #include <vm/vm_external.h>
103 #include <vm/memory_object.h>
104 #include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */
105 #include <vm/vm_shared_region.h>
106
107 #include <sys/codesign.h>
108 #include <sys/reason.h>
109 #include <sys/signalvar.h>
110
111 #include <san/kasan.h>
112
113 #define VM_FAULT_CLASSIFY 0
114
115 #define TRACEFAULTPAGE 0 /* (TEST/DEBUG) */
116
117 unsigned int vm_object_pagein_throttle = 16;
118
119 /*
120 * We apply a hard throttle to the demand zero rate of tasks that we believe are running out of control which
121 * kicks in when swap space runs out. 64-bit programs have massive address spaces and can leak enormous amounts
122 * of memory if they're buggy and can run the system completely out of swap space. If this happens, we
123 * impose a hard throttle on them to prevent them from taking the last bit of memory left. This helps
124 * keep the UI active so that the user has a chance to kill the offending task before the system
125 * completely hangs.
126 *
127 * The hard throttle is only applied when the system is nearly completely out of swap space and is only applied
128 * to tasks that appear to be bloated. When swap runs out, any task using more than vm_hard_throttle_threshold
129 * will be throttled. The throttling is done by giving the thread that's trying to demand zero a page a
130 * delay of HARD_THROTTLE_DELAY microseconds before being allowed to try the page fault again.
131 */
132
133 extern void throttle_lowpri_io(int);
134
135 extern struct vnode *vnode_pager_lookup_vnode(memory_object_t);
136
137 uint64_t vm_hard_throttle_threshold;
138
139
140
141 #define NEED_TO_HARD_THROTTLE_THIS_TASK() (vm_wants_task_throttled(current_task()) || \
142 ((vm_page_free_count < vm_page_throttle_limit || \
143 HARD_THROTTLE_LIMIT_REACHED()) && \
144 proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO) >= THROTTLE_LEVEL_THROTTLED))
145
146
147 #define HARD_THROTTLE_DELAY 10000 /* 10000 us == 10 ms */
148 #define SOFT_THROTTLE_DELAY 200 /* 200 us == .2 ms */
149
150 #define VM_PAGE_CREATION_THROTTLE_PERIOD_SECS 6
151 #define VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC 20000
152
153
154 boolean_t current_thread_aborted(void);
155
156 /* Forward declarations of internal routines. */
157 static kern_return_t vm_fault_wire_fast(
158 vm_map_t map,
159 vm_map_offset_t va,
160 vm_prot_t prot,
161 vm_tag_t wire_tag,
162 vm_map_entry_t entry,
163 pmap_t pmap,
164 vm_map_offset_t pmap_addr,
165 ppnum_t *physpage_p);
166
167 static kern_return_t vm_fault_internal(
168 vm_map_t map,
169 vm_map_offset_t vaddr,
170 vm_prot_t caller_prot,
171 boolean_t change_wiring,
172 vm_tag_t wire_tag,
173 int interruptible,
174 pmap_t pmap,
175 vm_map_offset_t pmap_addr,
176 ppnum_t *physpage_p);
177
178 static void vm_fault_copy_cleanup(
179 vm_page_t page,
180 vm_page_t top_page);
181
182 static void vm_fault_copy_dst_cleanup(
183 vm_page_t page);
184
185 #if VM_FAULT_CLASSIFY
186 extern void vm_fault_classify(vm_object_t object,
187 vm_object_offset_t offset,
188 vm_prot_t fault_type);
189
190 extern void vm_fault_classify_init(void);
191 #endif
192
193 unsigned long vm_pmap_enter_blocked = 0;
194 unsigned long vm_pmap_enter_retried = 0;
195
196 unsigned long vm_cs_validates = 0;
197 unsigned long vm_cs_revalidates = 0;
198 unsigned long vm_cs_query_modified = 0;
199 unsigned long vm_cs_validated_dirtied = 0;
200 unsigned long vm_cs_bitmap_validated = 0;
201 #if PMAP_CS
202 uint64_t vm_cs_defer_to_pmap_cs = 0;
203 uint64_t vm_cs_defer_to_pmap_cs_not = 0;
204 #endif /* PMAP_CS */
205
206 void vm_pre_fault(vm_map_offset_t);
207
208 extern char *kdp_compressor_decompressed_page;
209 extern addr64_t kdp_compressor_decompressed_page_paddr;
210 extern ppnum_t kdp_compressor_decompressed_page_ppnum;
211
212 struct vmrtfr {
213 int vmrtfr_maxi;
214 int vmrtfr_curi;
215 int64_t vmrtf_total;
216 vm_rtfault_record_t *vm_rtf_records;
217 } vmrtfrs;
218 #define VMRTF_DEFAULT_BUFSIZE (4096)
219 #define VMRTF_NUM_RECORDS_DEFAULT (VMRTF_DEFAULT_BUFSIZE / sizeof(vm_rtfault_record_t))
220 int vmrtf_num_records = VMRTF_NUM_RECORDS_DEFAULT;
221
222 static void vm_rtfrecord_lock(void);
223 static void vm_rtfrecord_unlock(void);
224 static void vm_record_rtfault(thread_t, uint64_t, vm_map_offset_t, int);
225
226 lck_spin_t vm_rtfr_slock;
227 extern lck_grp_t vm_page_lck_grp_bucket;
228 extern lck_attr_t vm_page_lck_attr;
229
230 /*
231 * Routine: vm_fault_init
232 * Purpose:
233 * Initialize our private data structures.
234 */
235 void
236 vm_fault_init(void)
237 {
238 int i, vm_compressor_temp;
239 boolean_t need_default_val = TRUE;
240 /*
241 * Choose a value for the hard throttle threshold based on the amount of ram. The threshold is
242 * computed as a percentage of available memory, and the percentage used is scaled inversely with
243 * the amount of memory. The percentage runs between 10% and 35%. We use 35% for small memory systems
244 * and reduce the value down to 10% for very large memory configurations. This helps give us a
245 * definition of a memory hog that makes more sense relative to the amount of ram in the machine.
246 * The formula here simply uses the number of gigabytes of ram to adjust the percentage.
247 */
248
249 vm_hard_throttle_threshold = sane_size * (35 - MIN((int)(sane_size / (1024*1024*1024)), 25)) / 100;
250
251 /*
252 * Configure compressed pager behavior. A boot arg takes precedence over a device tree entry.
253 */
254
255 if (PE_parse_boot_argn("vm_compressor", &vm_compressor_temp, sizeof (vm_compressor_temp))) {
256 for ( i = 0; i < VM_PAGER_MAX_MODES; i++) {
257 if (vm_compressor_temp > 0 &&
258 ((vm_compressor_temp & ( 1 << i)) == vm_compressor_temp)) {
259 need_default_val = FALSE;
260 vm_compressor_mode = vm_compressor_temp;
261 break;
262 }
263 }
264 if (need_default_val)
265 printf("Ignoring \"vm_compressor\" boot arg %d\n", vm_compressor_temp);
266 }
267 if (need_default_val) {
268 /* If no boot arg or incorrect boot arg, try device tree. */
269 PE_get_default("kern.vm_compressor", &vm_compressor_mode, sizeof(vm_compressor_mode));
270 }
271 printf("\"vm_compressor_mode\" is %d\n", vm_compressor_mode);
272 }
273
274 void vm_rtfault_record_init(void) {
275 PE_parse_boot_argn("vm_rtfault_records", &vmrtf_num_records, sizeof(vmrtf_num_records));
276
277 assert(vmrtf_num_records >= 1);
278 vmrtf_num_records = MAX(vmrtf_num_records, 1);
279 size_t kallocsz = vmrtf_num_records * sizeof(vm_rtfault_record_t);
280 vmrtfrs.vm_rtf_records = kalloc(kallocsz);
281 bzero(vmrtfrs.vm_rtf_records, kallocsz);
282 vmrtfrs.vmrtfr_maxi = vmrtf_num_records - 1;
283 lck_spin_init(&vm_rtfr_slock, &vm_page_lck_grp_bucket, &vm_page_lck_attr);
284 }
285 /*
286 * Routine: vm_fault_cleanup
287 * Purpose:
288 * Clean up the result of vm_fault_page.
289 * Results:
290 * The paging reference for "object" is released.
291 * "object" is unlocked.
292 * If "top_page" is not null, "top_page" is
293 * freed and the paging reference for the object
294 * containing it is released.
295 *
296 * In/out conditions:
297 * "object" must be locked.
298 */
299 void
300 vm_fault_cleanup(
301 vm_object_t object,
302 vm_page_t top_page)
303 {
304 vm_object_paging_end(object);
305 vm_object_unlock(object);
306
307 if (top_page != VM_PAGE_NULL) {
308 object = VM_PAGE_OBJECT(top_page);
309
310 vm_object_lock(object);
311 VM_PAGE_FREE(top_page);
312 vm_object_paging_end(object);
313 vm_object_unlock(object);
314 }
315 }
316
317 #define ALIGNED(x) (((x) & (PAGE_SIZE_64 - 1)) == 0)
318
319
320 boolean_t vm_page_deactivate_behind = TRUE;
321 /*
322 * default sizes given VM_BEHAVIOR_DEFAULT reference behavior
323 */
324 #define VM_DEFAULT_DEACTIVATE_BEHIND_WINDOW 128
325 #define VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER 16 /* don't make this too big... */
326 /* we use it to size an array on the stack */
327
328 int vm_default_behind = VM_DEFAULT_DEACTIVATE_BEHIND_WINDOW;
329
330 #define MAX_SEQUENTIAL_RUN (1024 * 1024 * 1024)
331
332 /*
333 * vm_page_is_sequential
334 *
335 * Determine if sequential access is in progress
336 * in accordance with the behavior specified.
337 * Update state to indicate current access pattern.
338 *
339 * object must have at least the shared lock held
340 */
341 static
342 void
343 vm_fault_is_sequential(
344 vm_object_t object,
345 vm_object_offset_t offset,
346 vm_behavior_t behavior)
347 {
348 vm_object_offset_t last_alloc;
349 int sequential;
350 int orig_sequential;
351
352 last_alloc = object->last_alloc;
353 sequential = object->sequential;
354 orig_sequential = sequential;
355
356 switch (behavior) {
357 case VM_BEHAVIOR_RANDOM:
358 /*
359 * reset indicator of sequential behavior
360 */
361 sequential = 0;
362 break;
363
364 case VM_BEHAVIOR_SEQUENTIAL:
365 if (offset && last_alloc == offset - PAGE_SIZE_64) {
366 /*
367 * advance indicator of sequential behavior
368 */
369 if (sequential < MAX_SEQUENTIAL_RUN)
370 sequential += PAGE_SIZE;
371 } else {
372 /*
373 * reset indicator of sequential behavior
374 */
375 sequential = 0;
376 }
377 break;
378
379 case VM_BEHAVIOR_RSEQNTL:
380 if (last_alloc && last_alloc == offset + PAGE_SIZE_64) {
381 /*
382 * advance indicator of sequential behavior
383 */
384 if (sequential > -MAX_SEQUENTIAL_RUN)
385 sequential -= PAGE_SIZE;
386 } else {
387 /*
388 * reset indicator of sequential behavior
389 */
390 sequential = 0;
391 }
392 break;
393
394 case VM_BEHAVIOR_DEFAULT:
395 default:
396 if (offset && last_alloc == (offset - PAGE_SIZE_64)) {
397 /*
398 * advance indicator of sequential behavior
399 */
400 if (sequential < 0)
401 sequential = 0;
402 if (sequential < MAX_SEQUENTIAL_RUN)
403 sequential += PAGE_SIZE;
404
405 } else if (last_alloc && last_alloc == (offset + PAGE_SIZE_64)) {
406 /*
407 * advance indicator of sequential behavior
408 */
409 if (sequential > 0)
410 sequential = 0;
411 if (sequential > -MAX_SEQUENTIAL_RUN)
412 sequential -= PAGE_SIZE;
413 } else {
414 /*
415 * reset indicator of sequential behavior
416 */
417 sequential = 0;
418 }
419 break;
420 }
421 if (sequential != orig_sequential) {
422 if (!OSCompareAndSwap(orig_sequential, sequential, (UInt32 *)&object->sequential)) {
423 /*
424 * if someone else has already updated object->sequential
425 * don't bother trying to update it or object->last_alloc
426 */
427 return;
428 }
429 }
430 /*
431 * I'd like to do this with a OSCompareAndSwap64, but that
432 * doesn't exist for PPC... however, it shouldn't matter
433 * that much... last_alloc is maintained so that we can determine
434 * if a sequential access pattern is taking place... if only
435 * one thread is banging on this object, no problem with the unprotected
436 * update... if 2 or more threads are banging away, we run the risk of
437 * someone seeing a mangled update... however, in the face of multiple
438 * accesses, no sequential access pattern can develop anyway, so we
439 * haven't lost any real info.
440 */
441 object->last_alloc = offset;
442 }
443
444
445 int vm_page_deactivate_behind_count = 0;
446
447 /*
448 * vm_page_deactivate_behind
449 *
450 * Determine if sequential access is in progress
451 * in accordance with the behavior specified. If
452 * so, compute a potential page to deactivate and
453 * deactivate it.
454 *
455 * object must be locked.
456 *
457 * return TRUE if we actually deactivate a page
458 */
459 static
460 boolean_t
461 vm_fault_deactivate_behind(
462 vm_object_t object,
463 vm_object_offset_t offset,
464 vm_behavior_t behavior)
465 {
466 int n;
467 int pages_in_run = 0;
468 int max_pages_in_run = 0;
469 int sequential_run;
470 int sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
471 vm_object_offset_t run_offset = 0;
472 vm_object_offset_t pg_offset = 0;
473 vm_page_t m;
474 vm_page_t page_run[VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER];
475
476 pages_in_run = 0;
477 #if TRACEFAULTPAGE
478 dbgTrace(0xBEEF0018, (unsigned int) object, (unsigned int) vm_fault_deactivate_behind); /* (TEST/DEBUG) */
479 #endif
480
481 if (object == kernel_object || vm_page_deactivate_behind == FALSE) {
482 /*
483 * Do not deactivate pages from the kernel object: they
484 * are not intended to become pageable.
485 * or we've disabled the deactivate behind mechanism
486 */
487 return FALSE;
488 }
489 if ((sequential_run = object->sequential)) {
490 if (sequential_run < 0) {
491 sequential_behavior = VM_BEHAVIOR_RSEQNTL;
492 sequential_run = 0 - sequential_run;
493 } else {
494 sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
495 }
496 }
497 switch (behavior) {
498 case VM_BEHAVIOR_RANDOM:
499 break;
500 case VM_BEHAVIOR_SEQUENTIAL:
501 if (sequential_run >= (int)PAGE_SIZE) {
502 run_offset = 0 - PAGE_SIZE_64;
503 max_pages_in_run = 1;
504 }
505 break;
506 case VM_BEHAVIOR_RSEQNTL:
507 if (sequential_run >= (int)PAGE_SIZE) {
508 run_offset = PAGE_SIZE_64;
509 max_pages_in_run = 1;
510 }
511 break;
512 case VM_BEHAVIOR_DEFAULT:
513 default:
514 { vm_object_offset_t behind = vm_default_behind * PAGE_SIZE_64;
515
516 /*
517 * determine if the run of sequential accesss has been
518 * long enough on an object with default access behavior
519 * to consider it for deactivation
520 */
521 if ((uint64_t)sequential_run >= behind && (sequential_run % (VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER * PAGE_SIZE)) == 0) {
522 /*
523 * the comparisons between offset and behind are done
524 * in this kind of odd fashion in order to prevent wrap around
525 * at the end points
526 */
527 if (sequential_behavior == VM_BEHAVIOR_SEQUENTIAL) {
528 if (offset >= behind) {
529 run_offset = 0 - behind;
530 pg_offset = PAGE_SIZE_64;
531 max_pages_in_run = VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER;
532 }
533 } else {
534 if (offset < -behind) {
535 run_offset = behind;
536 pg_offset = 0 - PAGE_SIZE_64;
537 max_pages_in_run = VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER;
538 }
539 }
540 }
541 break;
542 }
543 }
544 for (n = 0; n < max_pages_in_run; n++) {
545 m = vm_page_lookup(object, offset + run_offset + (n * pg_offset));
546
547 if (m && !m->vmp_laundry && !m->vmp_busy && !m->vmp_no_cache && (m->vmp_q_state != VM_PAGE_ON_THROTTLED_Q) && !m->vmp_fictitious && !m->vmp_absent) {
548 page_run[pages_in_run++] = m;
549
550 /*
551 * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise...
552 *
553 * a TLB flush isn't really needed here since at worst we'll miss the reference bit being
554 * updated in the PTE if a remote processor still has this mapping cached in its TLB when the
555 * new reference happens. If no futher references happen on the page after that remote TLB flushes
556 * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue
557 * by pageout_scan, which is just fine since the last reference would have happened quite far
558 * in the past (TLB caches don't hang around for very long), and of course could just as easily
559 * have happened before we did the deactivate_behind.
560 */
561 pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
562 }
563 }
564 if (pages_in_run) {
565 vm_page_lockspin_queues();
566
567 for (n = 0; n < pages_in_run; n++) {
568
569 m = page_run[n];
570
571 vm_page_deactivate_internal(m, FALSE);
572
573 vm_page_deactivate_behind_count++;
574 #if TRACEFAULTPAGE
575 dbgTrace(0xBEEF0019, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
576 #endif
577 }
578 vm_page_unlock_queues();
579
580 return TRUE;
581 }
582 return FALSE;
583 }
584
585
586 #if (DEVELOPMENT || DEBUG)
587 uint32_t vm_page_creation_throttled_hard = 0;
588 uint32_t vm_page_creation_throttled_soft = 0;
589 uint64_t vm_page_creation_throttle_avoided = 0;
590 #endif /* DEVELOPMENT || DEBUG */
591
592 static int
593 vm_page_throttled(boolean_t page_kept)
594 {
595 clock_sec_t elapsed_sec;
596 clock_sec_t tv_sec;
597 clock_usec_t tv_usec;
598
599 thread_t thread = current_thread();
600
601 if (thread->options & TH_OPT_VMPRIV)
602 return (0);
603
604 if (thread->t_page_creation_throttled) {
605 thread->t_page_creation_throttled = 0;
606
607 if (page_kept == FALSE)
608 goto no_throttle;
609 }
610 if (NEED_TO_HARD_THROTTLE_THIS_TASK()) {
611 #if (DEVELOPMENT || DEBUG)
612 thread->t_page_creation_throttled_hard++;
613 OSAddAtomic(1, &vm_page_creation_throttled_hard);
614 #endif /* DEVELOPMENT || DEBUG */
615 return (HARD_THROTTLE_DELAY);
616 }
617
618 if ((vm_page_free_count < vm_page_throttle_limit || (VM_CONFIG_COMPRESSOR_IS_PRESENT && SWAPPER_NEEDS_TO_UNTHROTTLE())) &&
619 thread->t_page_creation_count > (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS * VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC)) {
620
621 if (vm_page_free_wanted == 0 && vm_page_free_wanted_privileged == 0) {
622 #if (DEVELOPMENT || DEBUG)
623 OSAddAtomic64(1, &vm_page_creation_throttle_avoided);
624 #endif
625 goto no_throttle;
626 }
627 clock_get_system_microtime(&tv_sec, &tv_usec);
628
629 elapsed_sec = tv_sec - thread->t_page_creation_time;
630
631 if (elapsed_sec <= VM_PAGE_CREATION_THROTTLE_PERIOD_SECS ||
632 (thread->t_page_creation_count / elapsed_sec) >= VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC) {
633
634 if (elapsed_sec >= (3 * VM_PAGE_CREATION_THROTTLE_PERIOD_SECS)) {
635 /*
636 * we'll reset our stats to give a well behaved app
637 * that was unlucky enough to accumulate a bunch of pages
638 * over a long period of time a chance to get out of
639 * the throttled state... we reset the counter and timestamp
640 * so that if it stays under the rate limit for the next second
641 * it will be back in our good graces... if it exceeds it, it
642 * will remain in the throttled state
643 */
644 thread->t_page_creation_time = tv_sec;
645 thread->t_page_creation_count = VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC * (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS - 1);
646 }
647 VM_PAGEOUT_DEBUG(vm_page_throttle_count, 1);
648
649 thread->t_page_creation_throttled = 1;
650
651 if (VM_CONFIG_COMPRESSOR_IS_PRESENT && HARD_THROTTLE_LIMIT_REACHED()) {
652 #if (DEVELOPMENT || DEBUG)
653 thread->t_page_creation_throttled_hard++;
654 OSAddAtomic(1, &vm_page_creation_throttled_hard);
655 #endif /* DEVELOPMENT || DEBUG */
656 return (HARD_THROTTLE_DELAY);
657 } else {
658 #if (DEVELOPMENT || DEBUG)
659 thread->t_page_creation_throttled_soft++;
660 OSAddAtomic(1, &vm_page_creation_throttled_soft);
661 #endif /* DEVELOPMENT || DEBUG */
662 return (SOFT_THROTTLE_DELAY);
663 }
664 }
665 thread->t_page_creation_time = tv_sec;
666 thread->t_page_creation_count = 0;
667 }
668 no_throttle:
669 thread->t_page_creation_count++;
670
671 return (0);
672 }
673
674
675 /*
676 * check for various conditions that would
677 * prevent us from creating a ZF page...
678 * cleanup is based on being called from vm_fault_page
679 *
680 * object must be locked
681 * object == m->vmp_object
682 */
683 static vm_fault_return_t
684 vm_fault_check(vm_object_t object, vm_page_t m, vm_page_t first_m, wait_interrupt_t interruptible_state, boolean_t page_throttle)
685 {
686 int throttle_delay;
687
688 if (object->shadow_severed ||
689 VM_OBJECT_PURGEABLE_FAULT_ERROR(object)) {
690 /*
691 * Either:
692 * 1. the shadow chain was severed,
693 * 2. the purgeable object is volatile or empty and is marked
694 * to fault on access while volatile.
695 * Just have to return an error at this point
696 */
697 if (m != VM_PAGE_NULL)
698 VM_PAGE_FREE(m);
699 vm_fault_cleanup(object, first_m);
700
701 thread_interrupt_level(interruptible_state);
702
703 return (VM_FAULT_MEMORY_ERROR);
704 }
705 if (page_throttle == TRUE) {
706 if ((throttle_delay = vm_page_throttled(FALSE))) {
707 /*
708 * we're throttling zero-fills...
709 * treat this as if we couldn't grab a page
710 */
711 if (m != VM_PAGE_NULL)
712 VM_PAGE_FREE(m);
713 vm_fault_cleanup(object, first_m);
714
715 VM_DEBUG_EVENT(vmf_check_zfdelay, VMF_CHECK_ZFDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
716
717 delay(throttle_delay);
718
719 if (current_thread_aborted()) {
720 thread_interrupt_level(interruptible_state);
721 return VM_FAULT_INTERRUPTED;
722 }
723 thread_interrupt_level(interruptible_state);
724
725 return (VM_FAULT_MEMORY_SHORTAGE);
726 }
727 }
728 return (VM_FAULT_SUCCESS);
729 }
730
731
732 /*
733 * do the work to zero fill a page and
734 * inject it into the correct paging queue
735 *
736 * m->vmp_object must be locked
737 * page queue lock must NOT be held
738 */
739 static int
740 vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill)
741 {
742 int my_fault = DBG_ZERO_FILL_FAULT;
743 vm_object_t object;
744
745 object = VM_PAGE_OBJECT(m);
746
747 /*
748 * This is is a zero-fill page fault...
749 *
750 * Checking the page lock is a waste of
751 * time; this page was absent, so
752 * it can't be page locked by a pager.
753 *
754 * we also consider it undefined
755 * with respect to instruction
756 * execution. i.e. it is the responsibility
757 * of higher layers to call for an instruction
758 * sync after changing the contents and before
759 * sending a program into this area. We
760 * choose this approach for performance
761 */
762 m->vmp_pmapped = TRUE;
763
764 m->vmp_cs_validated = FALSE;
765 m->vmp_cs_tainted = FALSE;
766 m->vmp_cs_nx = FALSE;
767
768 if (no_zero_fill == TRUE) {
769 my_fault = DBG_NZF_PAGE_FAULT;
770
771 if (m->vmp_absent && m->vmp_busy)
772 return (my_fault);
773 } else {
774 vm_page_zero_fill(m);
775
776 VM_STAT_INCR(zero_fill_count);
777 DTRACE_VM2(zfod, int, 1, (uint64_t *), NULL);
778 }
779 assert(!m->vmp_laundry);
780 assert(object != kernel_object);
781 //assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
782
783 if (!VM_DYNAMIC_PAGING_ENABLED() &&
784 (object->purgable == VM_PURGABLE_DENY ||
785 object->purgable == VM_PURGABLE_NONVOLATILE ||
786 object->purgable == VM_PURGABLE_VOLATILE )) {
787
788 vm_page_lockspin_queues();
789
790 if (!VM_DYNAMIC_PAGING_ENABLED()) {
791 assert(!VM_PAGE_WIRED(m));
792
793 /*
794 * can't be on the pageout queue since we don't
795 * have a pager to try and clean to
796 */
797 vm_page_queues_remove(m, TRUE);
798 vm_page_check_pageable_safe(m);
799 vm_page_queue_enter(&vm_page_queue_throttled, m, vm_page_t, vmp_pageq);
800 m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
801 vm_page_throttled_count++;
802 }
803 vm_page_unlock_queues();
804 }
805 return (my_fault);
806 }
807
808
809 /*
810 * Routine: vm_fault_page
811 * Purpose:
812 * Find the resident page for the virtual memory
813 * specified by the given virtual memory object
814 * and offset.
815 * Additional arguments:
816 * The required permissions for the page is given
817 * in "fault_type". Desired permissions are included
818 * in "protection".
819 * fault_info is passed along to determine pagein cluster
820 * limits... it contains the expected reference pattern,
821 * cluster size if available, etc...
822 *
823 * If the desired page is known to be resident (for
824 * example, because it was previously wired down), asserting
825 * the "unwiring" parameter will speed the search.
826 *
827 * If the operation can be interrupted (by thread_abort
828 * or thread_terminate), then the "interruptible"
829 * parameter should be asserted.
830 *
831 * Results:
832 * The page containing the proper data is returned
833 * in "result_page".
834 *
835 * In/out conditions:
836 * The source object must be locked and referenced,
837 * and must donate one paging reference. The reference
838 * is not affected. The paging reference and lock are
839 * consumed.
840 *
841 * If the call succeeds, the object in which "result_page"
842 * resides is left locked and holding a paging reference.
843 * If this is not the original object, a busy page in the
844 * original object is returned in "top_page", to prevent other
845 * callers from pursuing this same data, along with a paging
846 * reference for the original object. The "top_page" should
847 * be destroyed when this guarantee is no longer required.
848 * The "result_page" is also left busy. It is not removed
849 * from the pageout queues.
850 * Special Case:
851 * A return value of VM_FAULT_SUCCESS_NO_PAGE means that the
852 * fault succeeded but there's no VM page (i.e. the VM object
853 * does not actually hold VM pages, but device memory or
854 * large pages). The object is still locked and we still hold a
855 * paging_in_progress reference.
856 */
857 unsigned int vm_fault_page_blocked_access = 0;
858 unsigned int vm_fault_page_forced_retry = 0;
859
860 vm_fault_return_t
861 vm_fault_page(
862 /* Arguments: */
863 vm_object_t first_object, /* Object to begin search */
864 vm_object_offset_t first_offset, /* Offset into object */
865 vm_prot_t fault_type, /* What access is requested */
866 boolean_t must_be_resident,/* Must page be resident? */
867 boolean_t caller_lookup, /* caller looked up page */
868 /* Modifies in place: */
869 vm_prot_t *protection, /* Protection for mapping */
870 vm_page_t *result_page, /* Page found, if successful */
871 /* Returns: */
872 vm_page_t *top_page, /* Page in top object, if
873 * not result_page. */
874 int *type_of_fault, /* if non-null, fill in with type of fault
875 * COW, zero-fill, etc... returned in trace point */
876 /* More arguments: */
877 kern_return_t *error_code, /* code if page is in error */
878 boolean_t no_zero_fill, /* don't zero fill absent pages */
879 boolean_t data_supply, /* treat as data_supply if
880 * it is a write fault and a full
881 * page is provided */
882 vm_object_fault_info_t fault_info)
883 {
884 vm_page_t m;
885 vm_object_t object;
886 vm_object_offset_t offset;
887 vm_page_t first_m;
888 vm_object_t next_object;
889 vm_object_t copy_object;
890 boolean_t look_for_page;
891 boolean_t force_fault_retry = FALSE;
892 vm_prot_t access_required = fault_type;
893 vm_prot_t wants_copy_flag;
894 kern_return_t wait_result;
895 wait_interrupt_t interruptible_state;
896 boolean_t data_already_requested = FALSE;
897 vm_behavior_t orig_behavior;
898 vm_size_t orig_cluster_size;
899 vm_fault_return_t error;
900 int my_fault;
901 uint32_t try_failed_count;
902 int interruptible; /* how may fault be interrupted? */
903 int external_state = VM_EXTERNAL_STATE_UNKNOWN;
904 memory_object_t pager;
905 vm_fault_return_t retval;
906 int grab_options;
907
908 /*
909 * MUST_ASK_PAGER() evaluates to TRUE if the page specified by object/offset is
910 * marked as paged out in the compressor pager or the pager doesn't exist.
911 * Note also that if the pager for an internal object
912 * has not been created, the pager is not invoked regardless of the value
913 * of MUST_ASK_PAGER().
914 *
915 * PAGED_OUT() evaluates to TRUE if the page specified by the object/offset
916 * is marked as paged out in the compressor pager.
917 * PAGED_OUT() is used to determine if a page has already been pushed
918 * into a copy object in order to avoid a redundant page out operation.
919 */
920 #define MUST_ASK_PAGER(o, f, s) \
921 ((s = VM_COMPRESSOR_PAGER_STATE_GET((o), (f))) != VM_EXTERNAL_STATE_ABSENT)
922
923 #define PAGED_OUT(o, f) \
924 (VM_COMPRESSOR_PAGER_STATE_GET((o), (f)) == VM_EXTERNAL_STATE_EXISTS)
925
926 /*
927 * Recovery actions
928 */
929 #define RELEASE_PAGE(m) \
930 MACRO_BEGIN \
931 PAGE_WAKEUP_DONE(m); \
932 if ( !VM_PAGE_PAGEABLE(m)) { \
933 vm_page_lockspin_queues(); \
934 if ( !VM_PAGE_PAGEABLE(m)) { \
935 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) \
936 vm_page_deactivate(m); \
937 else \
938 vm_page_activate(m); \
939 } \
940 vm_page_unlock_queues(); \
941 } \
942 MACRO_END
943
944 #if TRACEFAULTPAGE
945 dbgTrace(0xBEEF0002, (unsigned int) first_object, (unsigned int) first_offset); /* (TEST/DEBUG) */
946 #endif
947
948 interruptible = fault_info->interruptible;
949 interruptible_state = thread_interrupt_level(interruptible);
950
951 /*
952 * INVARIANTS (through entire routine):
953 *
954 * 1) At all times, we must either have the object
955 * lock or a busy page in some object to prevent
956 * some other thread from trying to bring in
957 * the same page.
958 *
959 * Note that we cannot hold any locks during the
960 * pager access or when waiting for memory, so
961 * we use a busy page then.
962 *
963 * 2) To prevent another thread from racing us down the
964 * shadow chain and entering a new page in the top
965 * object before we do, we must keep a busy page in
966 * the top object while following the shadow chain.
967 *
968 * 3) We must increment paging_in_progress on any object
969 * for which we have a busy page before dropping
970 * the object lock
971 *
972 * 4) We leave busy pages on the pageout queues.
973 * If the pageout daemon comes across a busy page,
974 * it will remove the page from the pageout queues.
975 */
976
977 object = first_object;
978 offset = first_offset;
979 first_m = VM_PAGE_NULL;
980 access_required = fault_type;
981
982
983 XPR(XPR_VM_FAULT,
984 "vm_f_page: obj 0x%X, offset 0x%X, type %d, prot %d\n",
985 object, offset, fault_type, *protection, 0);
986
987 /*
988 * default type of fault
989 */
990 my_fault = DBG_CACHE_HIT_FAULT;
991
992 while (TRUE) {
993 #if TRACEFAULTPAGE
994 dbgTrace(0xBEEF0003, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */
995 #endif
996
997 grab_options = 0;
998 #if CONFIG_SECLUDED_MEMORY
999 if (object->can_grab_secluded) {
1000 grab_options |= VM_PAGE_GRAB_SECLUDED;
1001 }
1002 #endif /* CONFIG_SECLUDED_MEMORY */
1003
1004 if (!object->alive) {
1005 /*
1006 * object is no longer valid
1007 * clean up and return error
1008 */
1009 vm_fault_cleanup(object, first_m);
1010 thread_interrupt_level(interruptible_state);
1011
1012 return (VM_FAULT_MEMORY_ERROR);
1013 }
1014
1015 if (!object->pager_created && object->phys_contiguous) {
1016 /*
1017 * A physically-contiguous object without a pager:
1018 * must be a "large page" object. We do not deal
1019 * with VM pages for this object.
1020 */
1021 caller_lookup = FALSE;
1022 m = VM_PAGE_NULL;
1023 goto phys_contig_object;
1024 }
1025
1026 if (object->blocked_access) {
1027 /*
1028 * Access to this VM object has been blocked.
1029 * Replace our "paging_in_progress" reference with
1030 * a "activity_in_progress" reference and wait for
1031 * access to be unblocked.
1032 */
1033 caller_lookup = FALSE; /* no longer valid after sleep */
1034 vm_object_activity_begin(object);
1035 vm_object_paging_end(object);
1036 while (object->blocked_access) {
1037 vm_object_sleep(object,
1038 VM_OBJECT_EVENT_UNBLOCKED,
1039 THREAD_UNINT);
1040 }
1041 vm_fault_page_blocked_access++;
1042 vm_object_paging_begin(object);
1043 vm_object_activity_end(object);
1044 }
1045
1046 /*
1047 * See whether the page at 'offset' is resident
1048 */
1049 if (caller_lookup == TRUE) {
1050 /*
1051 * The caller has already looked up the page
1052 * and gave us the result in "result_page".
1053 * We can use this for the first lookup but
1054 * it loses its validity as soon as we unlock
1055 * the object.
1056 */
1057 m = *result_page;
1058 caller_lookup = FALSE; /* no longer valid after that */
1059 } else {
1060 m = vm_page_lookup(object, offset);
1061 }
1062 #if TRACEFAULTPAGE
1063 dbgTrace(0xBEEF0004, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
1064 #endif
1065 if (m != VM_PAGE_NULL) {
1066
1067 if (m->vmp_busy) {
1068 /*
1069 * The page is being brought in,
1070 * wait for it and then retry.
1071 */
1072 #if TRACEFAULTPAGE
1073 dbgTrace(0xBEEF0005, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1074 #endif
1075 wait_result = PAGE_SLEEP(object, m, interruptible);
1076
1077 XPR(XPR_VM_FAULT,
1078 "vm_f_page: block busy obj 0x%X, offset 0x%X, page 0x%X\n",
1079 object, offset,
1080 m, 0, 0);
1081 counter(c_vm_fault_page_block_busy_kernel++);
1082
1083 if (wait_result != THREAD_AWAKENED) {
1084 vm_fault_cleanup(object, first_m);
1085 thread_interrupt_level(interruptible_state);
1086
1087 if (wait_result == THREAD_RESTART)
1088 return (VM_FAULT_RETRY);
1089 else
1090 return (VM_FAULT_INTERRUPTED);
1091 }
1092 continue;
1093 }
1094 if (m->vmp_laundry) {
1095 m->vmp_free_when_done = FALSE;
1096
1097 if (!m->vmp_cleaning)
1098 vm_pageout_steal_laundry(m, FALSE);
1099 }
1100 if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
1101 /*
1102 * Guard page: off limits !
1103 */
1104 if (fault_type == VM_PROT_NONE) {
1105 /*
1106 * The fault is not requesting any
1107 * access to the guard page, so it must
1108 * be just to wire or unwire it.
1109 * Let's pretend it succeeded...
1110 */
1111 m->vmp_busy = TRUE;
1112 *result_page = m;
1113 assert(first_m == VM_PAGE_NULL);
1114 *top_page = first_m;
1115 if (type_of_fault)
1116 *type_of_fault = DBG_GUARD_FAULT;
1117 thread_interrupt_level(interruptible_state);
1118 return VM_FAULT_SUCCESS;
1119 } else {
1120 /*
1121 * The fault requests access to the
1122 * guard page: let's deny that !
1123 */
1124 vm_fault_cleanup(object, first_m);
1125 thread_interrupt_level(interruptible_state);
1126 return VM_FAULT_MEMORY_ERROR;
1127 }
1128 }
1129
1130 if (m->vmp_error) {
1131 /*
1132 * The page is in error, give up now.
1133 */
1134 #if TRACEFAULTPAGE
1135 dbgTrace(0xBEEF0006, (unsigned int) m, (unsigned int) error_code); /* (TEST/DEBUG) */
1136 #endif
1137 if (error_code)
1138 *error_code = KERN_MEMORY_ERROR;
1139 VM_PAGE_FREE(m);
1140
1141 vm_fault_cleanup(object, first_m);
1142 thread_interrupt_level(interruptible_state);
1143
1144 return (VM_FAULT_MEMORY_ERROR);
1145 }
1146 if (m->vmp_restart) {
1147 /*
1148 * The pager wants us to restart
1149 * at the top of the chain,
1150 * typically because it has moved the
1151 * page to another pager, then do so.
1152 */
1153 #if TRACEFAULTPAGE
1154 dbgTrace(0xBEEF0007, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1155 #endif
1156 VM_PAGE_FREE(m);
1157
1158 vm_fault_cleanup(object, first_m);
1159 thread_interrupt_level(interruptible_state);
1160
1161 return (VM_FAULT_RETRY);
1162 }
1163 if (m->vmp_absent) {
1164 /*
1165 * The page isn't busy, but is absent,
1166 * therefore it's deemed "unavailable".
1167 *
1168 * Remove the non-existent page (unless it's
1169 * in the top object) and move on down to the
1170 * next object (if there is one).
1171 */
1172 #if TRACEFAULTPAGE
1173 dbgTrace(0xBEEF0008, (unsigned int) m, (unsigned int) object->shadow); /* (TEST/DEBUG) */
1174 #endif
1175 next_object = object->shadow;
1176
1177 if (next_object == VM_OBJECT_NULL) {
1178 /*
1179 * Absent page at bottom of shadow
1180 * chain; zero fill the page we left
1181 * busy in the first object, and free
1182 * the absent page.
1183 */
1184 assert(!must_be_resident);
1185
1186 /*
1187 * check for any conditions that prevent
1188 * us from creating a new zero-fill page
1189 * vm_fault_check will do all of the
1190 * fault cleanup in the case of an error condition
1191 * including resetting the thread_interrupt_level
1192 */
1193 error = vm_fault_check(object, m, first_m, interruptible_state, (type_of_fault == NULL) ? TRUE : FALSE);
1194
1195 if (error != VM_FAULT_SUCCESS)
1196 return (error);
1197
1198 XPR(XPR_VM_FAULT,
1199 "vm_f_page: zero obj 0x%X, off 0x%X, page 0x%X, first_obj 0x%X\n",
1200 object, offset,
1201 m,
1202 first_object, 0);
1203
1204 if (object != first_object) {
1205 /*
1206 * free the absent page we just found
1207 */
1208 VM_PAGE_FREE(m);
1209
1210 /*
1211 * drop reference and lock on current object
1212 */
1213 vm_object_paging_end(object);
1214 vm_object_unlock(object);
1215
1216 /*
1217 * grab the original page we
1218 * 'soldered' in place and
1219 * retake lock on 'first_object'
1220 */
1221 m = first_m;
1222 first_m = VM_PAGE_NULL;
1223
1224 object = first_object;
1225 offset = first_offset;
1226
1227 vm_object_lock(object);
1228 } else {
1229 /*
1230 * we're going to use the absent page we just found
1231 * so convert it to a 'busy' page
1232 */
1233 m->vmp_absent = FALSE;
1234 m->vmp_busy = TRUE;
1235 }
1236 if (fault_info->mark_zf_absent && no_zero_fill == TRUE)
1237 m->vmp_absent = TRUE;
1238 /*
1239 * zero-fill the page and put it on
1240 * the correct paging queue
1241 */
1242 my_fault = vm_fault_zero_page(m, no_zero_fill);
1243
1244 break;
1245 } else {
1246 if (must_be_resident)
1247 vm_object_paging_end(object);
1248 else if (object != first_object) {
1249 vm_object_paging_end(object);
1250 VM_PAGE_FREE(m);
1251 } else {
1252 first_m = m;
1253 m->vmp_absent = FALSE;
1254 m->vmp_busy = TRUE;
1255
1256 vm_page_lockspin_queues();
1257 vm_page_queues_remove(m, FALSE);
1258 vm_page_unlock_queues();
1259 }
1260 XPR(XPR_VM_FAULT,
1261 "vm_f_page: unavail obj 0x%X, off 0x%X, next_obj 0x%X, newoff 0x%X\n",
1262 object, offset,
1263 next_object,
1264 offset+object->vo_shadow_offset,0);
1265
1266 offset += object->vo_shadow_offset;
1267 fault_info->lo_offset += object->vo_shadow_offset;
1268 fault_info->hi_offset += object->vo_shadow_offset;
1269 access_required = VM_PROT_READ;
1270
1271 vm_object_lock(next_object);
1272 vm_object_unlock(object);
1273 object = next_object;
1274 vm_object_paging_begin(object);
1275
1276 /*
1277 * reset to default type of fault
1278 */
1279 my_fault = DBG_CACHE_HIT_FAULT;
1280
1281 continue;
1282 }
1283 }
1284 if ((m->vmp_cleaning)
1285 && ((object != first_object) || (object->copy != VM_OBJECT_NULL))
1286 && (fault_type & VM_PROT_WRITE)) {
1287 /*
1288 * This is a copy-on-write fault that will
1289 * cause us to revoke access to this page, but
1290 * this page is in the process of being cleaned
1291 * in a clustered pageout. We must wait until
1292 * the cleaning operation completes before
1293 * revoking access to the original page,
1294 * otherwise we might attempt to remove a
1295 * wired mapping.
1296 */
1297 #if TRACEFAULTPAGE
1298 dbgTrace(0xBEEF0009, (unsigned int) m, (unsigned int) offset); /* (TEST/DEBUG) */
1299 #endif
1300 XPR(XPR_VM_FAULT,
1301 "vm_f_page: cleaning obj 0x%X, offset 0x%X, page 0x%X\n",
1302 object, offset,
1303 m, 0, 0);
1304 /*
1305 * take an extra ref so that object won't die
1306 */
1307 vm_object_reference_locked(object);
1308
1309 vm_fault_cleanup(object, first_m);
1310
1311 counter(c_vm_fault_page_block_backoff_kernel++);
1312 vm_object_lock(object);
1313 assert(object->ref_count > 0);
1314
1315 m = vm_page_lookup(object, offset);
1316
1317 if (m != VM_PAGE_NULL && m->vmp_cleaning) {
1318 PAGE_ASSERT_WAIT(m, interruptible);
1319
1320 vm_object_unlock(object);
1321 wait_result = thread_block(THREAD_CONTINUE_NULL);
1322 vm_object_deallocate(object);
1323
1324 goto backoff;
1325 } else {
1326 vm_object_unlock(object);
1327
1328 vm_object_deallocate(object);
1329 thread_interrupt_level(interruptible_state);
1330
1331 return (VM_FAULT_RETRY);
1332 }
1333 }
1334 if (type_of_fault == NULL && (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) &&
1335 !(fault_info != NULL && fault_info->stealth)) {
1336 /*
1337 * If we were passed a non-NULL pointer for
1338 * "type_of_fault", than we came from
1339 * vm_fault... we'll let it deal with
1340 * this condition, since it
1341 * needs to see m->vmp_speculative to correctly
1342 * account the pageins, otherwise...
1343 * take it off the speculative queue, we'll
1344 * let the caller of vm_fault_page deal
1345 * with getting it onto the correct queue
1346 *
1347 * If the caller specified in fault_info that
1348 * it wants a "stealth" fault, we also leave
1349 * the page in the speculative queue.
1350 */
1351 vm_page_lockspin_queues();
1352 if (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q)
1353 vm_page_queues_remove(m, FALSE);
1354 vm_page_unlock_queues();
1355 }
1356 assert(object == VM_PAGE_OBJECT(m));
1357
1358 if (object->code_signed) {
1359 /*
1360 * CODE SIGNING:
1361 * We just paged in a page from a signed
1362 * memory object but we don't need to
1363 * validate it now. We'll validate it if
1364 * when it gets mapped into a user address
1365 * space for the first time or when the page
1366 * gets copied to another object as a result
1367 * of a copy-on-write.
1368 */
1369 }
1370
1371 /*
1372 * We mark the page busy and leave it on
1373 * the pageout queues. If the pageout
1374 * deamon comes across it, then it will
1375 * remove the page from the queue, but not the object
1376 */
1377 #if TRACEFAULTPAGE
1378 dbgTrace(0xBEEF000B, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1379 #endif
1380 XPR(XPR_VM_FAULT,
1381 "vm_f_page: found page obj 0x%X, offset 0x%X, page 0x%X\n",
1382 object, offset, m, 0, 0);
1383 assert(!m->vmp_busy);
1384 assert(!m->vmp_absent);
1385
1386 m->vmp_busy = TRUE;
1387 break;
1388 }
1389
1390
1391 /*
1392 * we get here when there is no page present in the object at
1393 * the offset we're interested in... we'll allocate a page
1394 * at this point if the pager associated with
1395 * this object can provide the data or we're the top object...
1396 * object is locked; m == NULL
1397 */
1398
1399 if (must_be_resident) {
1400 if (fault_type == VM_PROT_NONE &&
1401 object == kernel_object) {
1402 /*
1403 * We've been called from vm_fault_unwire()
1404 * while removing a map entry that was allocated
1405 * with KMA_KOBJECT and KMA_VAONLY. This page
1406 * is not present and there's nothing more to
1407 * do here (nothing to unwire).
1408 */
1409 vm_fault_cleanup(object, first_m);
1410 thread_interrupt_level(interruptible_state);
1411
1412 return VM_FAULT_MEMORY_ERROR;
1413 }
1414
1415 goto dont_look_for_page;
1416 }
1417
1418 /* Don't expect to fault pages into the kernel object. */
1419 assert(object != kernel_object);
1420
1421 data_supply = FALSE;
1422
1423 look_for_page = (object->pager_created && (MUST_ASK_PAGER(object, offset, external_state) == TRUE) && !data_supply);
1424
1425 #if TRACEFAULTPAGE
1426 dbgTrace(0xBEEF000C, (unsigned int) look_for_page, (unsigned int) object); /* (TEST/DEBUG) */
1427 #endif
1428 if (!look_for_page && object == first_object && !object->phys_contiguous) {
1429 /*
1430 * Allocate a new page for this object/offset pair as a placeholder
1431 */
1432 m = vm_page_grab_options(grab_options);
1433 #if TRACEFAULTPAGE
1434 dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
1435 #endif
1436 if (m == VM_PAGE_NULL) {
1437
1438 vm_fault_cleanup(object, first_m);
1439 thread_interrupt_level(interruptible_state);
1440
1441 return (VM_FAULT_MEMORY_SHORTAGE);
1442 }
1443
1444 if (fault_info && fault_info->batch_pmap_op == TRUE) {
1445 vm_page_insert_internal(m, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, TRUE, FALSE, NULL);
1446 } else {
1447 vm_page_insert(m, object, offset);
1448 }
1449 }
1450 if (look_for_page) {
1451 kern_return_t rc;
1452 int my_fault_type;
1453
1454 /*
1455 * If the memory manager is not ready, we
1456 * cannot make requests.
1457 */
1458 if (!object->pager_ready) {
1459 #if TRACEFAULTPAGE
1460 dbgTrace(0xBEEF000E, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */
1461 #endif
1462 if (m != VM_PAGE_NULL)
1463 VM_PAGE_FREE(m);
1464
1465 XPR(XPR_VM_FAULT,
1466 "vm_f_page: ready wait obj 0x%X, offset 0x%X\n",
1467 object, offset, 0, 0, 0);
1468
1469 /*
1470 * take an extra ref so object won't die
1471 */
1472 vm_object_reference_locked(object);
1473 vm_fault_cleanup(object, first_m);
1474 counter(c_vm_fault_page_block_backoff_kernel++);
1475
1476 vm_object_lock(object);
1477 assert(object->ref_count > 0);
1478
1479 if (!object->pager_ready) {
1480 wait_result = vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGER_READY, interruptible);
1481
1482 vm_object_unlock(object);
1483 if (wait_result == THREAD_WAITING)
1484 wait_result = thread_block(THREAD_CONTINUE_NULL);
1485 vm_object_deallocate(object);
1486
1487 goto backoff;
1488 } else {
1489 vm_object_unlock(object);
1490 vm_object_deallocate(object);
1491 thread_interrupt_level(interruptible_state);
1492
1493 return (VM_FAULT_RETRY);
1494 }
1495 }
1496 if (!object->internal && !object->phys_contiguous && object->paging_in_progress > vm_object_pagein_throttle) {
1497 /*
1498 * If there are too many outstanding page
1499 * requests pending on this external object, we
1500 * wait for them to be resolved now.
1501 */
1502 #if TRACEFAULTPAGE
1503 dbgTrace(0xBEEF0010, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1504 #endif
1505 if (m != VM_PAGE_NULL)
1506 VM_PAGE_FREE(m);
1507 /*
1508 * take an extra ref so object won't die
1509 */
1510 vm_object_reference_locked(object);
1511
1512 vm_fault_cleanup(object, first_m);
1513
1514 counter(c_vm_fault_page_block_backoff_kernel++);
1515
1516 vm_object_lock(object);
1517 assert(object->ref_count > 0);
1518
1519 if (object->paging_in_progress >= vm_object_pagein_throttle) {
1520 vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS, interruptible);
1521
1522 vm_object_unlock(object);
1523 wait_result = thread_block(THREAD_CONTINUE_NULL);
1524 vm_object_deallocate(object);
1525
1526 goto backoff;
1527 } else {
1528 vm_object_unlock(object);
1529 vm_object_deallocate(object);
1530 thread_interrupt_level(interruptible_state);
1531
1532 return (VM_FAULT_RETRY);
1533 }
1534 }
1535 if (object->internal) {
1536 int compressed_count_delta;
1537
1538 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
1539
1540 if (m == VM_PAGE_NULL) {
1541 /*
1542 * Allocate a new page for this object/offset pair as a placeholder
1543 */
1544 m = vm_page_grab_options(grab_options);
1545 #if TRACEFAULTPAGE
1546 dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
1547 #endif
1548 if (m == VM_PAGE_NULL) {
1549
1550 vm_fault_cleanup(object, first_m);
1551 thread_interrupt_level(interruptible_state);
1552
1553 return (VM_FAULT_MEMORY_SHORTAGE);
1554 }
1555
1556 m->vmp_absent = TRUE;
1557 if (fault_info && fault_info->batch_pmap_op == TRUE) {
1558 vm_page_insert_internal(m, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, TRUE, FALSE, NULL);
1559 } else {
1560 vm_page_insert(m, object, offset);
1561 }
1562 }
1563 assert(m->vmp_busy);
1564
1565 m->vmp_absent = TRUE;
1566 pager = object->pager;
1567
1568 assert(object->paging_in_progress > 0);
1569 vm_object_unlock(object);
1570
1571 rc = vm_compressor_pager_get(
1572 pager,
1573 offset + object->paging_offset,
1574 VM_PAGE_GET_PHYS_PAGE(m),
1575 &my_fault_type,
1576 0,
1577 &compressed_count_delta);
1578
1579 if (type_of_fault == NULL) {
1580 int throttle_delay;
1581
1582 /*
1583 * we weren't called from vm_fault, so we
1584 * need to apply page creation throttling
1585 * do it before we re-acquire any locks
1586 */
1587 if (my_fault_type == DBG_COMPRESSOR_FAULT) {
1588 if ((throttle_delay = vm_page_throttled(TRUE))) {
1589 VM_DEBUG_EVENT(vmf_compressordelay, VMF_COMPRESSORDELAY, DBG_FUNC_NONE, throttle_delay, 0, 1, 0);
1590 delay(throttle_delay);
1591 }
1592 }
1593 }
1594 vm_object_lock(object);
1595 assert(object->paging_in_progress > 0);
1596
1597 vm_compressor_pager_count(
1598 pager,
1599 compressed_count_delta,
1600 FALSE, /* shared_lock */
1601 object);
1602
1603 switch (rc) {
1604 case KERN_SUCCESS:
1605 m->vmp_absent = FALSE;
1606 m->vmp_dirty = TRUE;
1607 if ((object->wimg_bits &
1608 VM_WIMG_MASK) !=
1609 VM_WIMG_USE_DEFAULT) {
1610 /*
1611 * If the page is not cacheable,
1612 * we can't let its contents
1613 * linger in the data cache
1614 * after the decompression.
1615 */
1616 pmap_sync_page_attributes_phys(
1617 VM_PAGE_GET_PHYS_PAGE(m));
1618 } else {
1619 m->vmp_written_by_kernel = TRUE;
1620 }
1621
1622 /*
1623 * If the object is purgeable, its
1624 * owner's purgeable ledgers have been
1625 * updated in vm_page_insert() but the
1626 * page was also accounted for in a
1627 * "compressed purgeable" ledger, so
1628 * update that now.
1629 */
1630 if (((object->purgable !=
1631 VM_PURGABLE_DENY) ||
1632 object->vo_ledger_tag) &&
1633 (object->vo_owner !=
1634 NULL)) {
1635 /*
1636 * One less compressed
1637 * purgeable/tagged page.
1638 */
1639 vm_object_owner_compressed_update(
1640 object,
1641 -1);
1642 }
1643
1644 break;
1645 case KERN_MEMORY_FAILURE:
1646 m->vmp_unusual = TRUE;
1647 m->vmp_error = TRUE;
1648 m->vmp_absent = FALSE;
1649 break;
1650 case KERN_MEMORY_ERROR:
1651 assert(m->vmp_absent);
1652 break;
1653 default:
1654 panic("vm_fault_page(): unexpected "
1655 "error %d from "
1656 "vm_compressor_pager_get()\n",
1657 rc);
1658 }
1659 PAGE_WAKEUP_DONE(m);
1660
1661 rc = KERN_SUCCESS;
1662 goto data_requested;
1663 }
1664 my_fault_type = DBG_PAGEIN_FAULT;
1665
1666 if (m != VM_PAGE_NULL) {
1667 VM_PAGE_FREE(m);
1668 m = VM_PAGE_NULL;
1669 }
1670
1671 #if TRACEFAULTPAGE
1672 dbgTrace(0xBEEF0012, (unsigned int) object, (unsigned int) 0); /* (TEST/DEBUG) */
1673 #endif
1674
1675 /*
1676 * It's possible someone called vm_object_destroy while we weren't
1677 * holding the object lock. If that has happened, then bail out
1678 * here.
1679 */
1680
1681 pager = object->pager;
1682
1683 if (pager == MEMORY_OBJECT_NULL) {
1684 vm_fault_cleanup(object, first_m);
1685 thread_interrupt_level(interruptible_state);
1686 return VM_FAULT_MEMORY_ERROR;
1687 }
1688
1689 /*
1690 * We have an absent page in place for the faulting offset,
1691 * so we can release the object lock.
1692 */
1693
1694 if (object->object_is_shared_cache) {
1695 set_thread_rwlock_boost();
1696 }
1697
1698 vm_object_unlock(object);
1699
1700 /*
1701 * If this object uses a copy_call strategy,
1702 * and we are interested in a copy of this object
1703 * (having gotten here only by following a
1704 * shadow chain), then tell the memory manager
1705 * via a flag added to the desired_access
1706 * parameter, so that it can detect a race
1707 * between our walking down the shadow chain
1708 * and its pushing pages up into a copy of
1709 * the object that it manages.
1710 */
1711 if (object->copy_strategy == MEMORY_OBJECT_COPY_CALL && object != first_object)
1712 wants_copy_flag = VM_PROT_WANTS_COPY;
1713 else
1714 wants_copy_flag = VM_PROT_NONE;
1715
1716 XPR(XPR_VM_FAULT,
1717 "vm_f_page: data_req obj 0x%X, offset 0x%X, page 0x%X, acc %d\n",
1718 object, offset, m,
1719 access_required | wants_copy_flag, 0);
1720
1721 if (object->copy == first_object) {
1722 /*
1723 * if we issue the memory_object_data_request in
1724 * this state, we are subject to a deadlock with
1725 * the underlying filesystem if it is trying to
1726 * shrink the file resulting in a push of pages
1727 * into the copy object... that push will stall
1728 * on the placeholder page, and if the pushing thread
1729 * is holding a lock that is required on the pagein
1730 * path (such as a truncate lock), we'll deadlock...
1731 * to avoid this potential deadlock, we throw away
1732 * our placeholder page before calling memory_object_data_request
1733 * and force this thread to retry the vm_fault_page after
1734 * we have issued the I/O. the second time through this path
1735 * we will find the page already in the cache (presumably still
1736 * busy waiting for the I/O to complete) and then complete
1737 * the fault w/o having to go through memory_object_data_request again
1738 */
1739 assert(first_m != VM_PAGE_NULL);
1740 assert(VM_PAGE_OBJECT(first_m) == first_object);
1741
1742 vm_object_lock(first_object);
1743 VM_PAGE_FREE(first_m);
1744 vm_object_paging_end(first_object);
1745 vm_object_unlock(first_object);
1746
1747 first_m = VM_PAGE_NULL;
1748 force_fault_retry = TRUE;
1749
1750 vm_fault_page_forced_retry++;
1751 }
1752
1753 if (data_already_requested == TRUE) {
1754 orig_behavior = fault_info->behavior;
1755 orig_cluster_size = fault_info->cluster_size;
1756
1757 fault_info->behavior = VM_BEHAVIOR_RANDOM;
1758 fault_info->cluster_size = PAGE_SIZE;
1759 }
1760 /*
1761 * Call the memory manager to retrieve the data.
1762 */
1763 rc = memory_object_data_request(
1764 pager,
1765 offset + object->paging_offset,
1766 PAGE_SIZE,
1767 access_required | wants_copy_flag,
1768 (memory_object_fault_info_t)fault_info);
1769
1770 if (data_already_requested == TRUE) {
1771 fault_info->behavior = orig_behavior;
1772 fault_info->cluster_size = orig_cluster_size;
1773 } else
1774 data_already_requested = TRUE;
1775
1776 DTRACE_VM2(maj_fault, int, 1, (uint64_t *), NULL);
1777 #if TRACEFAULTPAGE
1778 dbgTrace(0xBEEF0013, (unsigned int) object, (unsigned int) rc); /* (TEST/DEBUG) */
1779 #endif
1780 vm_object_lock(object);
1781
1782 if (object->object_is_shared_cache) {
1783 clear_thread_rwlock_boost();
1784 }
1785
1786 data_requested:
1787 if (rc != KERN_SUCCESS) {
1788
1789 vm_fault_cleanup(object, first_m);
1790 thread_interrupt_level(interruptible_state);
1791
1792 return ((rc == MACH_SEND_INTERRUPTED) ?
1793 VM_FAULT_INTERRUPTED :
1794 VM_FAULT_MEMORY_ERROR);
1795 } else {
1796 clock_sec_t tv_sec;
1797 clock_usec_t tv_usec;
1798
1799 if (my_fault_type == DBG_PAGEIN_FAULT) {
1800 clock_get_system_microtime(&tv_sec, &tv_usec);
1801 current_thread()->t_page_creation_time = tv_sec;
1802 current_thread()->t_page_creation_count = 0;
1803 }
1804 }
1805 if ((interruptible != THREAD_UNINT) && (current_thread()->sched_flags & TH_SFLAG_ABORT)) {
1806
1807 vm_fault_cleanup(object, first_m);
1808 thread_interrupt_level(interruptible_state);
1809
1810 return (VM_FAULT_INTERRUPTED);
1811 }
1812 if (force_fault_retry == TRUE) {
1813
1814 vm_fault_cleanup(object, first_m);
1815 thread_interrupt_level(interruptible_state);
1816
1817 return (VM_FAULT_RETRY);
1818 }
1819 if (m == VM_PAGE_NULL && object->phys_contiguous) {
1820 /*
1821 * No page here means that the object we
1822 * initially looked up was "physically
1823 * contiguous" (i.e. device memory). However,
1824 * with Virtual VRAM, the object might not
1825 * be backed by that device memory anymore,
1826 * so we're done here only if the object is
1827 * still "phys_contiguous".
1828 * Otherwise, if the object is no longer
1829 * "phys_contiguous", we need to retry the
1830 * page fault against the object's new backing
1831 * store (different memory object).
1832 */
1833 phys_contig_object:
1834 goto done;
1835 }
1836 /*
1837 * potentially a pagein fault
1838 * if we make it through the state checks
1839 * above, than we'll count it as such
1840 */
1841 my_fault = my_fault_type;
1842
1843 /*
1844 * Retry with same object/offset, since new data may
1845 * be in a different page (i.e., m is meaningless at
1846 * this point).
1847 */
1848 continue;
1849 }
1850 dont_look_for_page:
1851 /*
1852 * We get here if the object has no pager, or an existence map
1853 * exists and indicates the page isn't present on the pager
1854 * or we're unwiring a page. If a pager exists, but there
1855 * is no existence map, then the m->vmp_absent case above handles
1856 * the ZF case when the pager can't provide the page
1857 */
1858 #if TRACEFAULTPAGE
1859 dbgTrace(0xBEEF0014, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
1860 #endif
1861 if (object == first_object)
1862 first_m = m;
1863 else
1864 assert(m == VM_PAGE_NULL);
1865
1866 XPR(XPR_VM_FAULT,
1867 "vm_f_page: no pager obj 0x%X, offset 0x%X, page 0x%X, next_obj 0x%X\n",
1868 object, offset, m,
1869 object->shadow, 0);
1870
1871 next_object = object->shadow;
1872
1873 if (next_object == VM_OBJECT_NULL) {
1874 /*
1875 * we've hit the bottom of the shadown chain,
1876 * fill the page in the top object with zeros.
1877 */
1878 assert(!must_be_resident);
1879
1880 if (object != first_object) {
1881 vm_object_paging_end(object);
1882 vm_object_unlock(object);
1883
1884 object = first_object;
1885 offset = first_offset;
1886 vm_object_lock(object);
1887 }
1888 m = first_m;
1889 assert(VM_PAGE_OBJECT(m) == object);
1890 first_m = VM_PAGE_NULL;
1891
1892 /*
1893 * check for any conditions that prevent
1894 * us from creating a new zero-fill page
1895 * vm_fault_check will do all of the
1896 * fault cleanup in the case of an error condition
1897 * including resetting the thread_interrupt_level
1898 */
1899 error = vm_fault_check(object, m, first_m, interruptible_state, (type_of_fault == NULL) ? TRUE : FALSE);
1900
1901 if (error != VM_FAULT_SUCCESS)
1902 return (error);
1903
1904 if (m == VM_PAGE_NULL) {
1905 m = vm_page_grab_options(grab_options);
1906
1907 if (m == VM_PAGE_NULL) {
1908 vm_fault_cleanup(object, VM_PAGE_NULL);
1909 thread_interrupt_level(interruptible_state);
1910
1911 return (VM_FAULT_MEMORY_SHORTAGE);
1912 }
1913 vm_page_insert(m, object, offset);
1914 }
1915 if (fault_info->mark_zf_absent && no_zero_fill == TRUE)
1916 m->vmp_absent = TRUE;
1917
1918 my_fault = vm_fault_zero_page(m, no_zero_fill);
1919
1920 break;
1921
1922 } else {
1923 /*
1924 * Move on to the next object. Lock the next
1925 * object before unlocking the current one.
1926 */
1927 if ((object != first_object) || must_be_resident)
1928 vm_object_paging_end(object);
1929
1930 offset += object->vo_shadow_offset;
1931 fault_info->lo_offset += object->vo_shadow_offset;
1932 fault_info->hi_offset += object->vo_shadow_offset;
1933 access_required = VM_PROT_READ;
1934
1935 vm_object_lock(next_object);
1936 vm_object_unlock(object);
1937
1938 object = next_object;
1939 vm_object_paging_begin(object);
1940 }
1941 }
1942
1943 /*
1944 * PAGE HAS BEEN FOUND.
1945 *
1946 * This page (m) is:
1947 * busy, so that we can play with it;
1948 * not absent, so that nobody else will fill it;
1949 * possibly eligible for pageout;
1950 *
1951 * The top-level page (first_m) is:
1952 * VM_PAGE_NULL if the page was found in the
1953 * top-level object;
1954 * busy, not absent, and ineligible for pageout.
1955 *
1956 * The current object (object) is locked. A paging
1957 * reference is held for the current and top-level
1958 * objects.
1959 */
1960
1961 #if TRACEFAULTPAGE
1962 dbgTrace(0xBEEF0015, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
1963 #endif
1964 #if EXTRA_ASSERTIONS
1965 assert(m->vmp_busy && !m->vmp_absent);
1966 assert((first_m == VM_PAGE_NULL) ||
1967 (first_m->vmp_busy && !first_m->vmp_absent &&
1968 !first_m->vmp_active && !first_m->vmp_inactive && !first_m->vmp_secluded));
1969 #endif /* EXTRA_ASSERTIONS */
1970
1971 XPR(XPR_VM_FAULT,
1972 "vm_f_page: FOUND obj 0x%X, off 0x%X, page 0x%X, 1_obj 0x%X, 1_m 0x%X\n",
1973 object, offset, m,
1974 first_object, first_m);
1975
1976 /*
1977 * If the page is being written, but isn't
1978 * already owned by the top-level object,
1979 * we have to copy it into a new page owned
1980 * by the top-level object.
1981 */
1982 if (object != first_object) {
1983
1984 #if TRACEFAULTPAGE
1985 dbgTrace(0xBEEF0016, (unsigned int) object, (unsigned int) fault_type); /* (TEST/DEBUG) */
1986 #endif
1987 if (fault_type & VM_PROT_WRITE) {
1988 vm_page_t copy_m;
1989
1990 /*
1991 * We only really need to copy if we
1992 * want to write it.
1993 */
1994 assert(!must_be_resident);
1995
1996 /*
1997 * If we try to collapse first_object at this
1998 * point, we may deadlock when we try to get
1999 * the lock on an intermediate object (since we
2000 * have the bottom object locked). We can't
2001 * unlock the bottom object, because the page
2002 * we found may move (by collapse) if we do.
2003 *
2004 * Instead, we first copy the page. Then, when
2005 * we have no more use for the bottom object,
2006 * we unlock it and try to collapse.
2007 *
2008 * Note that we copy the page even if we didn't
2009 * need to... that's the breaks.
2010 */
2011
2012 /*
2013 * Allocate a page for the copy
2014 */
2015 copy_m = vm_page_grab_options(grab_options);
2016
2017 if (copy_m == VM_PAGE_NULL) {
2018 RELEASE_PAGE(m);
2019
2020 vm_fault_cleanup(object, first_m);
2021 thread_interrupt_level(interruptible_state);
2022
2023 return (VM_FAULT_MEMORY_SHORTAGE);
2024 }
2025 XPR(XPR_VM_FAULT,
2026 "vm_f_page: page_copy obj 0x%X, offset 0x%X, m 0x%X, copy_m 0x%X\n",
2027 object, offset,
2028 m, copy_m, 0);
2029
2030 vm_page_copy(m, copy_m);
2031
2032 /*
2033 * If another map is truly sharing this
2034 * page with us, we have to flush all
2035 * uses of the original page, since we
2036 * can't distinguish those which want the
2037 * original from those which need the
2038 * new copy.
2039 *
2040 * XXXO If we know that only one map has
2041 * access to this page, then we could
2042 * avoid the pmap_disconnect() call.
2043 */
2044 if (m->vmp_pmapped)
2045 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
2046
2047 if (m->vmp_clustered) {
2048 VM_PAGE_COUNT_AS_PAGEIN(m);
2049 VM_PAGE_CONSUME_CLUSTERED(m);
2050 }
2051 assert(!m->vmp_cleaning);
2052
2053 /*
2054 * We no longer need the old page or object.
2055 */
2056 RELEASE_PAGE(m);
2057
2058 /*
2059 * This check helps with marking the object as having a sequential pattern
2060 * Normally we'll miss doing this below because this fault is about COW to
2061 * the first_object i.e. bring page in from disk, push to object above but
2062 * don't update the file object's sequential pattern.
2063 */
2064 if (object->internal == FALSE) {
2065 vm_fault_is_sequential(object, offset, fault_info->behavior);
2066 }
2067
2068 vm_object_paging_end(object);
2069 vm_object_unlock(object);
2070
2071 my_fault = DBG_COW_FAULT;
2072 VM_STAT_INCR(cow_faults);
2073 DTRACE_VM2(cow_fault, int, 1, (uint64_t *), NULL);
2074 current_task()->cow_faults++;
2075
2076 object = first_object;
2077 offset = first_offset;
2078
2079 vm_object_lock(object);
2080 /*
2081 * get rid of the place holder
2082 * page that we soldered in earlier
2083 */
2084 VM_PAGE_FREE(first_m);
2085 first_m = VM_PAGE_NULL;
2086
2087 /*
2088 * and replace it with the
2089 * page we just copied into
2090 */
2091 assert(copy_m->vmp_busy);
2092 vm_page_insert(copy_m, object, offset);
2093 SET_PAGE_DIRTY(copy_m, TRUE);
2094
2095 m = copy_m;
2096 /*
2097 * Now that we've gotten the copy out of the
2098 * way, let's try to collapse the top object.
2099 * But we have to play ugly games with
2100 * paging_in_progress to do that...
2101 */
2102 vm_object_paging_end(object);
2103 vm_object_collapse(object, offset, TRUE);
2104 vm_object_paging_begin(object);
2105
2106 } else
2107 *protection &= (~VM_PROT_WRITE);
2108 }
2109 /*
2110 * Now check whether the page needs to be pushed into the
2111 * copy object. The use of asymmetric copy on write for
2112 * shared temporary objects means that we may do two copies to
2113 * satisfy the fault; one above to get the page from a
2114 * shadowed object, and one here to push it into the copy.
2115 */
2116 try_failed_count = 0;
2117
2118 while ((copy_object = first_object->copy) != VM_OBJECT_NULL) {
2119 vm_object_offset_t copy_offset;
2120 vm_page_t copy_m;
2121
2122 #if TRACEFAULTPAGE
2123 dbgTrace(0xBEEF0017, (unsigned int) copy_object, (unsigned int) fault_type); /* (TEST/DEBUG) */
2124 #endif
2125 /*
2126 * If the page is being written, but hasn't been
2127 * copied to the copy-object, we have to copy it there.
2128 */
2129 if ((fault_type & VM_PROT_WRITE) == 0) {
2130 *protection &= ~VM_PROT_WRITE;
2131 break;
2132 }
2133
2134 /*
2135 * If the page was guaranteed to be resident,
2136 * we must have already performed the copy.
2137 */
2138 if (must_be_resident)
2139 break;
2140
2141 /*
2142 * Try to get the lock on the copy_object.
2143 */
2144 if (!vm_object_lock_try(copy_object)) {
2145
2146 vm_object_unlock(object);
2147 try_failed_count++;
2148
2149 mutex_pause(try_failed_count); /* wait a bit */
2150 vm_object_lock(object);
2151
2152 continue;
2153 }
2154 try_failed_count = 0;
2155
2156 /*
2157 * Make another reference to the copy-object,
2158 * to keep it from disappearing during the
2159 * copy.
2160 */
2161 vm_object_reference_locked(copy_object);
2162
2163 /*
2164 * Does the page exist in the copy?
2165 */
2166 copy_offset = first_offset - copy_object->vo_shadow_offset;
2167
2168 if (copy_object->vo_size <= copy_offset)
2169 /*
2170 * Copy object doesn't cover this page -- do nothing.
2171 */
2172 ;
2173 else if ((copy_m = vm_page_lookup(copy_object, copy_offset)) != VM_PAGE_NULL) {
2174 /*
2175 * Page currently exists in the copy object
2176 */
2177 if (copy_m->vmp_busy) {
2178 /*
2179 * If the page is being brought
2180 * in, wait for it and then retry.
2181 */
2182 RELEASE_PAGE(m);
2183
2184 /*
2185 * take an extra ref so object won't die
2186 */
2187 vm_object_reference_locked(copy_object);
2188 vm_object_unlock(copy_object);
2189 vm_fault_cleanup(object, first_m);
2190 counter(c_vm_fault_page_block_backoff_kernel++);
2191
2192 vm_object_lock(copy_object);
2193 assert(copy_object->ref_count > 0);
2194 VM_OBJ_RES_DECR(copy_object);
2195 vm_object_lock_assert_exclusive(copy_object);
2196 copy_object->ref_count--;
2197 assert(copy_object->ref_count > 0);
2198 copy_m = vm_page_lookup(copy_object, copy_offset);
2199
2200 if (copy_m != VM_PAGE_NULL && copy_m->vmp_busy) {
2201 PAGE_ASSERT_WAIT(copy_m, interruptible);
2202
2203 vm_object_unlock(copy_object);
2204 wait_result = thread_block(THREAD_CONTINUE_NULL);
2205 vm_object_deallocate(copy_object);
2206
2207 goto backoff;
2208 } else {
2209 vm_object_unlock(copy_object);
2210 vm_object_deallocate(copy_object);
2211 thread_interrupt_level(interruptible_state);
2212
2213 return (VM_FAULT_RETRY);
2214 }
2215 }
2216 }
2217 else if (!PAGED_OUT(copy_object, copy_offset)) {
2218 /*
2219 * If PAGED_OUT is TRUE, then the page used to exist
2220 * in the copy-object, and has already been paged out.
2221 * We don't need to repeat this. If PAGED_OUT is
2222 * FALSE, then either we don't know (!pager_created,
2223 * for example) or it hasn't been paged out.
2224 * (VM_EXTERNAL_STATE_UNKNOWN||VM_EXTERNAL_STATE_ABSENT)
2225 * We must copy the page to the copy object.
2226 *
2227 * Allocate a page for the copy
2228 */
2229 copy_m = vm_page_alloc(copy_object, copy_offset);
2230
2231 if (copy_m == VM_PAGE_NULL) {
2232 RELEASE_PAGE(m);
2233
2234 VM_OBJ_RES_DECR(copy_object);
2235 vm_object_lock_assert_exclusive(copy_object);
2236 copy_object->ref_count--;
2237 assert(copy_object->ref_count > 0);
2238
2239 vm_object_unlock(copy_object);
2240 vm_fault_cleanup(object, first_m);
2241 thread_interrupt_level(interruptible_state);
2242
2243 return (VM_FAULT_MEMORY_SHORTAGE);
2244 }
2245 /*
2246 * Must copy page into copy-object.
2247 */
2248 vm_page_copy(m, copy_m);
2249
2250 /*
2251 * If the old page was in use by any users
2252 * of the copy-object, it must be removed
2253 * from all pmaps. (We can't know which
2254 * pmaps use it.)
2255 */
2256 if (m->vmp_pmapped)
2257 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
2258
2259 if (m->vmp_clustered) {
2260 VM_PAGE_COUNT_AS_PAGEIN(m);
2261 VM_PAGE_CONSUME_CLUSTERED(m);
2262 }
2263 /*
2264 * If there's a pager, then immediately
2265 * page out this page, using the "initialize"
2266 * option. Else, we use the copy.
2267 */
2268 if ((!copy_object->pager_ready)
2269 || VM_COMPRESSOR_PAGER_STATE_GET(copy_object, copy_offset) == VM_EXTERNAL_STATE_ABSENT
2270 ) {
2271
2272 vm_page_lockspin_queues();
2273 assert(!m->vmp_cleaning);
2274 vm_page_activate(copy_m);
2275 vm_page_unlock_queues();
2276
2277 SET_PAGE_DIRTY(copy_m, TRUE);
2278 PAGE_WAKEUP_DONE(copy_m);
2279
2280 } else {
2281
2282 assert(copy_m->vmp_busy == TRUE);
2283 assert(!m->vmp_cleaning);
2284
2285 /*
2286 * dirty is protected by the object lock
2287 */
2288 SET_PAGE_DIRTY(copy_m, TRUE);
2289
2290 /*
2291 * The page is already ready for pageout:
2292 * not on pageout queues and busy.
2293 * Unlock everything except the
2294 * copy_object itself.
2295 */
2296 vm_object_unlock(object);
2297
2298 /*
2299 * Write the page to the copy-object,
2300 * flushing it from the kernel.
2301 */
2302 vm_pageout_initialize_page(copy_m);
2303
2304 /*
2305 * Since the pageout may have
2306 * temporarily dropped the
2307 * copy_object's lock, we
2308 * check whether we'll have
2309 * to deallocate the hard way.
2310 */
2311 if ((copy_object->shadow != object) || (copy_object->ref_count == 1)) {
2312 vm_object_unlock(copy_object);
2313 vm_object_deallocate(copy_object);
2314 vm_object_lock(object);
2315
2316 continue;
2317 }
2318 /*
2319 * Pick back up the old object's
2320 * lock. [It is safe to do so,
2321 * since it must be deeper in the
2322 * object tree.]
2323 */
2324 vm_object_lock(object);
2325 }
2326
2327 /*
2328 * Because we're pushing a page upward
2329 * in the object tree, we must restart
2330 * any faults that are waiting here.
2331 * [Note that this is an expansion of
2332 * PAGE_WAKEUP that uses the THREAD_RESTART
2333 * wait result]. Can't turn off the page's
2334 * busy bit because we're not done with it.
2335 */
2336 if (m->vmp_wanted) {
2337 m->vmp_wanted = FALSE;
2338 thread_wakeup_with_result((event_t) m, THREAD_RESTART);
2339 }
2340 }
2341 /*
2342 * The reference count on copy_object must be
2343 * at least 2: one for our extra reference,
2344 * and at least one from the outside world
2345 * (we checked that when we last locked
2346 * copy_object).
2347 */
2348 vm_object_lock_assert_exclusive(copy_object);
2349 copy_object->ref_count--;
2350 assert(copy_object->ref_count > 0);
2351
2352 VM_OBJ_RES_DECR(copy_object);
2353 vm_object_unlock(copy_object);
2354
2355 break;
2356 }
2357
2358 done:
2359 *result_page = m;
2360 *top_page = first_m;
2361
2362 XPR(XPR_VM_FAULT,
2363 "vm_f_page: DONE obj 0x%X, offset 0x%X, m 0x%X, first_m 0x%X\n",
2364 object, offset, m, first_m, 0);
2365
2366 if (m != VM_PAGE_NULL) {
2367 assert(VM_PAGE_OBJECT(m) == object);
2368
2369 retval = VM_FAULT_SUCCESS;
2370
2371 if (my_fault == DBG_PAGEIN_FAULT) {
2372
2373 VM_PAGE_COUNT_AS_PAGEIN(m);
2374
2375 if (object->internal)
2376 my_fault = DBG_PAGEIND_FAULT;
2377 else
2378 my_fault = DBG_PAGEINV_FAULT;
2379
2380 /*
2381 * evaluate access pattern and update state
2382 * vm_fault_deactivate_behind depends on the
2383 * state being up to date
2384 */
2385 vm_fault_is_sequential(object, offset, fault_info->behavior);
2386 vm_fault_deactivate_behind(object, offset, fault_info->behavior);
2387
2388 } else if (type_of_fault == NULL && my_fault == DBG_CACHE_HIT_FAULT) {
2389 /*
2390 * we weren't called from vm_fault, so handle the
2391 * accounting here for hits in the cache
2392 */
2393 if (m->vmp_clustered) {
2394 VM_PAGE_COUNT_AS_PAGEIN(m);
2395 VM_PAGE_CONSUME_CLUSTERED(m);
2396 }
2397 vm_fault_is_sequential(object, offset, fault_info->behavior);
2398 vm_fault_deactivate_behind(object, offset, fault_info->behavior);
2399
2400 } else if (my_fault == DBG_COMPRESSOR_FAULT || my_fault == DBG_COMPRESSOR_SWAPIN_FAULT) {
2401
2402 VM_STAT_INCR(decompressions);
2403 }
2404 if (type_of_fault)
2405 *type_of_fault = my_fault;
2406 } else {
2407 retval = VM_FAULT_SUCCESS_NO_VM_PAGE;
2408 assert(first_m == VM_PAGE_NULL);
2409 assert(object == first_object);
2410 }
2411
2412 thread_interrupt_level(interruptible_state);
2413
2414 #if TRACEFAULTPAGE
2415 dbgTrace(0xBEEF001A, (unsigned int) VM_FAULT_SUCCESS, 0); /* (TEST/DEBUG) */
2416 #endif
2417 return retval;
2418
2419 backoff:
2420 thread_interrupt_level(interruptible_state);
2421
2422 if (wait_result == THREAD_INTERRUPTED)
2423 return (VM_FAULT_INTERRUPTED);
2424 return (VM_FAULT_RETRY);
2425
2426 #undef RELEASE_PAGE
2427 }
2428
2429
2430
2431 /*
2432 * CODE SIGNING:
2433 * When soft faulting a page, we have to validate the page if:
2434 * 1. the page is being mapped in user space
2435 * 2. the page hasn't already been found to be "tainted"
2436 * 3. the page belongs to a code-signed object
2437 * 4. the page has not been validated yet or has been mapped for write.
2438 */
2439 #define VM_FAULT_NEED_CS_VALIDATION(pmap, page, page_obj) \
2440 ((pmap) != kernel_pmap /*1*/ && \
2441 !(page)->vmp_cs_tainted /*2*/ && \
2442 (page_obj)->code_signed /*3*/ && \
2443 (!(page)->vmp_cs_validated || (page)->vmp_wpmapped /*4*/))
2444
2445
2446 /*
2447 * page queue lock must NOT be held
2448 * m->vmp_object must be locked
2449 *
2450 * NOTE: m->vmp_object could be locked "shared" only if we are called
2451 * from vm_fault() as part of a soft fault. If so, we must be
2452 * careful not to modify the VM object in any way that is not
2453 * legal under a shared lock...
2454 */
2455 extern int panic_on_cs_killed;
2456 extern int proc_selfpid(void);
2457 extern char *proc_name_address(void *p);
2458 unsigned long cs_enter_tainted_rejected = 0;
2459 unsigned long cs_enter_tainted_accepted = 0;
2460 kern_return_t
2461 vm_fault_enter(vm_page_t m,
2462 pmap_t pmap,
2463 vm_map_offset_t vaddr,
2464 vm_prot_t prot,
2465 vm_prot_t caller_prot,
2466 boolean_t wired,
2467 boolean_t change_wiring,
2468 vm_tag_t wire_tag,
2469 vm_object_fault_info_t fault_info,
2470 boolean_t *need_retry,
2471 int *type_of_fault)
2472 {
2473 kern_return_t kr, pe_result;
2474 boolean_t previously_pmapped = m->vmp_pmapped;
2475 boolean_t must_disconnect = 0;
2476 boolean_t map_is_switched, map_is_switch_protected;
2477 boolean_t cs_violation;
2478 int cs_enforcement_enabled;
2479 vm_prot_t fault_type;
2480 vm_object_t object;
2481 boolean_t no_cache = fault_info->no_cache;
2482 boolean_t cs_bypass = fault_info->cs_bypass;
2483 int pmap_options = fault_info->pmap_options;
2484
2485 fault_type = change_wiring ? VM_PROT_NONE : caller_prot;
2486 object = VM_PAGE_OBJECT(m);
2487
2488 vm_object_lock_assert_held(object);
2489
2490 #if KASAN
2491 if (pmap == kernel_pmap) {
2492 kasan_notify_address(vaddr, PAGE_SIZE);
2493 }
2494 #endif
2495
2496 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
2497
2498 if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
2499 assert(m->vmp_fictitious);
2500 return KERN_SUCCESS;
2501 }
2502
2503 if (*type_of_fault == DBG_ZERO_FILL_FAULT) {
2504
2505 vm_object_lock_assert_exclusive(object);
2506
2507 } else if ((fault_type & VM_PROT_WRITE) == 0 &&
2508 (!m->vmp_wpmapped
2509 #if VM_OBJECT_ACCESS_TRACKING
2510 || object->access_tracking
2511 #endif /* VM_OBJECT_ACCESS_TRACKING */
2512 )) {
2513 /*
2514 * This is not a "write" fault, so we
2515 * might not have taken the object lock
2516 * exclusively and we might not be able
2517 * to update the "wpmapped" bit in
2518 * vm_fault_enter().
2519 * Let's just grant read access to
2520 * the page for now and we'll
2521 * soft-fault again if we need write
2522 * access later...
2523 */
2524
2525 /* This had better not be a JIT page. */
2526 if (!pmap_has_prot_policy(prot)) {
2527 prot &= ~VM_PROT_WRITE;
2528 } else {
2529 assert(cs_bypass);
2530 }
2531 }
2532 if (m->vmp_pmapped == FALSE) {
2533
2534 if (m->vmp_clustered) {
2535 if (*type_of_fault == DBG_CACHE_HIT_FAULT) {
2536 /*
2537 * found it in the cache, but this
2538 * is the first fault-in of the page (m->vmp_pmapped == FALSE)
2539 * so it must have come in as part of
2540 * a cluster... account 1 pagein against it
2541 */
2542 if (object->internal)
2543 *type_of_fault = DBG_PAGEIND_FAULT;
2544 else
2545 *type_of_fault = DBG_PAGEINV_FAULT;
2546
2547 VM_PAGE_COUNT_AS_PAGEIN(m);
2548 }
2549 VM_PAGE_CONSUME_CLUSTERED(m);
2550 }
2551 }
2552
2553 if (*type_of_fault != DBG_COW_FAULT) {
2554 DTRACE_VM2(as_fault, int, 1, (uint64_t *), NULL);
2555
2556 if (pmap == kernel_pmap) {
2557 DTRACE_VM2(kernel_asflt, int, 1, (uint64_t *), NULL);
2558 }
2559 }
2560
2561 /* Validate code signature if necessary. */
2562 if (!cs_bypass &&
2563 VM_FAULT_NEED_CS_VALIDATION(pmap, m, object)) {
2564 vm_object_lock_assert_exclusive(object);
2565
2566 if (m->vmp_cs_validated) {
2567 vm_cs_revalidates++;
2568 }
2569
2570 /* VM map is locked, so 1 ref will remain on VM object -
2571 * so no harm if vm_page_validate_cs drops the object lock */
2572
2573 #if PMAP_CS
2574 if (fault_info->pmap_cs_associated &&
2575 pmap_cs_enforced(pmap) &&
2576 !m->vmp_cs_validated &&
2577 !m->vmp_cs_tainted &&
2578 !m->vmp_cs_nx &&
2579 (prot & VM_PROT_EXECUTE) &&
2580 (caller_prot & VM_PROT_EXECUTE)) {
2581 /*
2582 * With pmap_cs, the pmap layer will validate the
2583 * code signature for any executable pmap mapping.
2584 * No need for us to validate this page too:
2585 * in pmap_cs we trust...
2586 */
2587 vm_cs_defer_to_pmap_cs++;
2588 } else {
2589 vm_cs_defer_to_pmap_cs_not++;
2590 vm_page_validate_cs(m);
2591 }
2592 #else /* PMAP_CS */
2593 vm_page_validate_cs(m);
2594 #endif /* PMAP_CS */
2595 }
2596
2597 #define page_immutable(m,prot) ((m)->vmp_cs_validated /*&& ((prot) & VM_PROT_EXECUTE)*/)
2598 #define page_nx(m) ((m)->vmp_cs_nx)
2599
2600 map_is_switched = ((pmap != vm_map_pmap(current_task()->map)) &&
2601 (pmap == vm_map_pmap(current_thread()->map)));
2602 map_is_switch_protected = current_thread()->map->switch_protect;
2603
2604 /* If the map is switched, and is switch-protected, we must protect
2605 * some pages from being write-faulted: immutable pages because by
2606 * definition they may not be written, and executable pages because that
2607 * would provide a way to inject unsigned code.
2608 * If the page is immutable, we can simply return. However, we can't
2609 * immediately determine whether a page is executable anywhere. But,
2610 * we can disconnect it everywhere and remove the executable protection
2611 * from the current map. We do that below right before we do the
2612 * PMAP_ENTER.
2613 */
2614 cs_enforcement_enabled = cs_process_enforcement(NULL);
2615
2616 if(cs_enforcement_enabled && map_is_switched &&
2617 map_is_switch_protected && page_immutable(m, prot) &&
2618 (prot & VM_PROT_WRITE))
2619 {
2620 return KERN_CODESIGN_ERROR;
2621 }
2622
2623 if (cs_enforcement_enabled && page_nx(m) && (prot & VM_PROT_EXECUTE)) {
2624 if (cs_debug)
2625 printf("page marked to be NX, not letting it be mapped EXEC\n");
2626 return KERN_CODESIGN_ERROR;
2627 }
2628
2629 /* A page could be tainted, or pose a risk of being tainted later.
2630 * Check whether the receiving process wants it, and make it feel
2631 * the consequences (that hapens in cs_invalid_page()).
2632 * For CS Enforcement, two other conditions will
2633 * cause that page to be tainted as well:
2634 * - pmapping an unsigned page executable - this means unsigned code;
2635 * - writeable mapping of a validated page - the content of that page
2636 * can be changed without the kernel noticing, therefore unsigned
2637 * code can be created
2638 */
2639 if (cs_bypass) {
2640 /* code-signing is bypassed */
2641 cs_violation = FALSE;
2642 } else if (m->vmp_cs_tainted) {
2643 /* tainted page */
2644 cs_violation = TRUE;
2645 } else if (!cs_enforcement_enabled) {
2646 /* no further code-signing enforcement */
2647 cs_violation = FALSE;
2648 } else if (page_immutable(m, prot) &&
2649 ((prot & VM_PROT_WRITE) ||
2650 m->vmp_wpmapped)) {
2651 /*
2652 * The page should be immutable, but is in danger of being
2653 * modified.
2654 * This is the case where we want policy from the code
2655 * directory - is the page immutable or not? For now we have
2656 * to assume that code pages will be immutable, data pages not.
2657 * We'll assume a page is a code page if it has a code directory
2658 * and we fault for execution.
2659 * That is good enough since if we faulted the code page for
2660 * writing in another map before, it is wpmapped; if we fault
2661 * it for writing in this map later it will also be faulted for
2662 * executing at the same time; and if we fault for writing in
2663 * another map later, we will disconnect it from this pmap so
2664 * we'll notice the change.
2665 */
2666 cs_violation = TRUE;
2667 } else if (!m->vmp_cs_validated &&
2668 (prot & VM_PROT_EXECUTE)
2669 #if PMAP_CS
2670 /*
2671 * Executable pages will be validated by pmap_cs;
2672 * in pmap_cs we trust...
2673 * If pmap_cs is turned off, this is a code-signing
2674 * violation.
2675 */
2676 && ! (pmap_cs_enforced(pmap))
2677 #endif /* PMAP_CS */
2678 ) {
2679 cs_violation = TRUE;
2680 } else {
2681 cs_violation = FALSE;
2682 }
2683
2684 if (cs_violation) {
2685 /* We will have a tainted page. Have to handle the special case
2686 * of a switched map now. If the map is not switched, standard
2687 * procedure applies - call cs_invalid_page().
2688 * If the map is switched, the real owner is invalid already.
2689 * There is no point in invalidating the switching process since
2690 * it will not be executing from the map. So we don't call
2691 * cs_invalid_page() in that case. */
2692 boolean_t reject_page, cs_killed;
2693 if(map_is_switched) {
2694 assert(pmap==vm_map_pmap(current_thread()->map));
2695 assert(!(prot & VM_PROT_WRITE) || (map_is_switch_protected == FALSE));
2696 reject_page = FALSE;
2697 } else {
2698 if (cs_debug > 5)
2699 printf("vm_fault: signed: %s validate: %s tainted: %s wpmapped: %s prot: 0x%x\n",
2700 object->code_signed ? "yes" : "no",
2701 m->vmp_cs_validated ? "yes" : "no",
2702 m->vmp_cs_tainted ? "yes" : "no",
2703 m->vmp_wpmapped ? "yes" : "no",
2704 (int)prot);
2705 reject_page = cs_invalid_page((addr64_t) vaddr, &cs_killed);
2706 }
2707
2708 if (reject_page) {
2709 /* reject the invalid page: abort the page fault */
2710 int pid;
2711 const char *procname;
2712 task_t task;
2713 vm_object_t file_object, shadow;
2714 vm_object_offset_t file_offset;
2715 char *pathname, *filename;
2716 vm_size_t pathname_len, filename_len;
2717 boolean_t truncated_path;
2718 #define __PATH_MAX 1024
2719 struct timespec mtime, cs_mtime;
2720 int shadow_depth;
2721 os_reason_t codesigning_exit_reason = OS_REASON_NULL;
2722
2723 kr = KERN_CODESIGN_ERROR;
2724 cs_enter_tainted_rejected++;
2725
2726 /* get process name and pid */
2727 procname = "?";
2728 task = current_task();
2729 pid = proc_selfpid();
2730 if (task->bsd_info != NULL)
2731 procname = proc_name_address(task->bsd_info);
2732
2733 /* get file's VM object */
2734 file_object = object;
2735 file_offset = m->vmp_offset;
2736 for (shadow = file_object->shadow,
2737 shadow_depth = 0;
2738 shadow != VM_OBJECT_NULL;
2739 shadow = file_object->shadow,
2740 shadow_depth++) {
2741 vm_object_lock_shared(shadow);
2742 if (file_object != object) {
2743 vm_object_unlock(file_object);
2744 }
2745 file_offset += file_object->vo_shadow_offset;
2746 file_object = shadow;
2747 }
2748
2749 mtime.tv_sec = 0;
2750 mtime.tv_nsec = 0;
2751 cs_mtime.tv_sec = 0;
2752 cs_mtime.tv_nsec = 0;
2753
2754 /* get file's pathname and/or filename */
2755 pathname = NULL;
2756 filename = NULL;
2757 pathname_len = 0;
2758 filename_len = 0;
2759 truncated_path = FALSE;
2760 /* no pager -> no file -> no pathname, use "<nil>" in that case */
2761 if (file_object->pager != NULL) {
2762 pathname = (char *)kalloc(__PATH_MAX * 2);
2763 if (pathname) {
2764 pathname[0] = '\0';
2765 pathname_len = __PATH_MAX;
2766 filename = pathname + pathname_len;
2767 filename_len = __PATH_MAX;
2768 }
2769 vnode_pager_get_object_name(file_object->pager,
2770 pathname,
2771 pathname_len,
2772 filename,
2773 filename_len,
2774 &truncated_path);
2775 if (pathname) {
2776 /* safety first... */
2777 pathname[__PATH_MAX-1] = '\0';
2778 filename[__PATH_MAX-1] = '\0';
2779 }
2780 vnode_pager_get_object_mtime(file_object->pager,
2781 &mtime,
2782 &cs_mtime);
2783 }
2784 printf("CODE SIGNING: process %d[%s]: "
2785 "rejecting invalid page at address 0x%llx "
2786 "from offset 0x%llx in file \"%s%s%s\" "
2787 "(cs_mtime:%lu.%ld %s mtime:%lu.%ld) "
2788 "(signed:%d validated:%d tainted:%d nx:%d "
2789 "wpmapped:%d dirty:%d depth:%d)\n",
2790 pid, procname, (addr64_t) vaddr,
2791 file_offset,
2792 (pathname ? pathname : "<nil>"),
2793 (truncated_path ? "/.../" : ""),
2794 (truncated_path ? filename : ""),
2795 cs_mtime.tv_sec, cs_mtime.tv_nsec,
2796 ((cs_mtime.tv_sec == mtime.tv_sec &&
2797 cs_mtime.tv_nsec == mtime.tv_nsec)
2798 ? "=="
2799 : "!="),
2800 mtime.tv_sec, mtime.tv_nsec,
2801 object->code_signed,
2802 m->vmp_cs_validated,
2803 m->vmp_cs_tainted,
2804 m->vmp_cs_nx,
2805 m->vmp_wpmapped,
2806 m->vmp_dirty,
2807 shadow_depth);
2808
2809 /*
2810 * We currently only generate an exit reason if cs_invalid_page directly killed a process. If cs_invalid_page
2811 * did not kill the process (more the case on desktop), vm_fault_enter will not satisfy the fault and whether the
2812 * process dies is dependent on whether there is a signal handler registered for SIGSEGV and how that handler
2813 * will deal with the segmentation fault.
2814 */
2815 if (cs_killed) {
2816 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXITREASON_CREATE) | DBG_FUNC_NONE,
2817 pid, OS_REASON_CODESIGNING, CODESIGNING_EXIT_REASON_INVALID_PAGE, 0, 0);
2818
2819 codesigning_exit_reason = os_reason_create(OS_REASON_CODESIGNING, CODESIGNING_EXIT_REASON_INVALID_PAGE);
2820 if (codesigning_exit_reason == NULL) {
2821 printf("vm_fault_enter: failed to allocate codesigning exit reason\n");
2822 } else {
2823 mach_vm_address_t data_addr = 0;
2824 struct codesigning_exit_reason_info *ceri = NULL;
2825 uint32_t reason_buffer_size_estimate = kcdata_estimate_required_buffer_size(1, sizeof(*ceri));
2826
2827 if (os_reason_alloc_buffer_noblock(codesigning_exit_reason, reason_buffer_size_estimate)) {
2828 printf("vm_fault_enter: failed to allocate buffer for codesigning exit reason\n");
2829 } else {
2830 if (KERN_SUCCESS == kcdata_get_memory_addr(&codesigning_exit_reason->osr_kcd_descriptor,
2831 EXIT_REASON_CODESIGNING_INFO, sizeof(*ceri), &data_addr)) {
2832 ceri = (struct codesigning_exit_reason_info *)data_addr;
2833 static_assert(__PATH_MAX == sizeof(ceri->ceri_pathname));
2834
2835 ceri->ceri_virt_addr = vaddr;
2836 ceri->ceri_file_offset = file_offset;
2837 if (pathname)
2838 strncpy((char *)&ceri->ceri_pathname, pathname, sizeof(ceri->ceri_pathname));
2839 else
2840 ceri->ceri_pathname[0] = '\0';
2841 if (filename)
2842 strncpy((char *)&ceri->ceri_filename, filename, sizeof(ceri->ceri_filename));
2843 else
2844 ceri->ceri_filename[0] = '\0';
2845 ceri->ceri_path_truncated = (truncated_path);
2846 ceri->ceri_codesig_modtime_secs = cs_mtime.tv_sec;
2847 ceri->ceri_codesig_modtime_nsecs = cs_mtime.tv_nsec;
2848 ceri->ceri_page_modtime_secs = mtime.tv_sec;
2849 ceri->ceri_page_modtime_nsecs = mtime.tv_nsec;
2850 ceri->ceri_object_codesigned = (object->code_signed);
2851 ceri->ceri_page_codesig_validated = (m->vmp_cs_validated);
2852 ceri->ceri_page_codesig_tainted = (m->vmp_cs_tainted);
2853 ceri->ceri_page_codesig_nx = (m->vmp_cs_nx);
2854 ceri->ceri_page_wpmapped = (m->vmp_wpmapped);
2855 ceri->ceri_page_slid = 0;
2856 ceri->ceri_page_dirty = (m->vmp_dirty);
2857 ceri->ceri_page_shadow_depth = shadow_depth;
2858 } else {
2859 #if DEBUG || DEVELOPMENT
2860 panic("vm_fault_enter: failed to allocate kcdata for codesigning exit reason");
2861 #else
2862 printf("vm_fault_enter: failed to allocate kcdata for codesigning exit reason\n");
2863 #endif /* DEBUG || DEVELOPMENT */
2864 /* Free the buffer */
2865 os_reason_alloc_buffer_noblock(codesigning_exit_reason, 0);
2866 }
2867 }
2868 }
2869
2870 set_thread_exit_reason(current_thread(), codesigning_exit_reason, FALSE);
2871 }
2872 if (panic_on_cs_killed &&
2873 object->object_is_shared_cache) {
2874 panic("CODE SIGNING: process %d[%s]: "
2875 "rejecting invalid page at address 0x%llx "
2876 "from offset 0x%llx in file \"%s%s%s\" "
2877 "(cs_mtime:%lu.%ld %s mtime:%lu.%ld) "
2878 "(signed:%d validated:%d tainted:%d nx:%d"
2879 "wpmapped:%d dirty:%d depth:%d)\n",
2880 pid, procname, (addr64_t) vaddr,
2881 file_offset,
2882 (pathname ? pathname : "<nil>"),
2883 (truncated_path ? "/.../" : ""),
2884 (truncated_path ? filename : ""),
2885 cs_mtime.tv_sec, cs_mtime.tv_nsec,
2886 ((cs_mtime.tv_sec == mtime.tv_sec &&
2887 cs_mtime.tv_nsec == mtime.tv_nsec)
2888 ? "=="
2889 : "!="),
2890 mtime.tv_sec, mtime.tv_nsec,
2891 object->code_signed,
2892 m->vmp_cs_validated,
2893 m->vmp_cs_tainted,
2894 m->vmp_cs_nx,
2895 m->vmp_wpmapped,
2896 m->vmp_dirty,
2897 shadow_depth);
2898 }
2899
2900 if (file_object != object) {
2901 vm_object_unlock(file_object);
2902 }
2903 if (pathname_len != 0) {
2904 kfree(pathname, __PATH_MAX * 2);
2905 pathname = NULL;
2906 filename = NULL;
2907 }
2908 } else {
2909 /* proceed with the invalid page */
2910 kr = KERN_SUCCESS;
2911 if (!m->vmp_cs_validated &&
2912 !object->code_signed) {
2913 /*
2914 * This page has not been (fully) validated but
2915 * does not belong to a code-signed object
2916 * so it should not be forcefully considered
2917 * as tainted.
2918 * We're just concerned about it here because
2919 * we've been asked to "execute" it but that
2920 * does not mean that it should cause other
2921 * accesses to fail.
2922 * This happens when a debugger sets a
2923 * breakpoint and we then execute code in
2924 * that page. Marking the page as "tainted"
2925 * would cause any inspection tool ("leaks",
2926 * "vmmap", "CrashReporter", ...) to get killed
2927 * due to code-signing violation on that page,
2928 * even though they're just reading it and not
2929 * executing from it.
2930 */
2931 } else {
2932 /*
2933 * Page might have been tainted before or not;
2934 * now it definitively is. If the page wasn't
2935 * tainted, we must disconnect it from all
2936 * pmaps later, to force existing mappings
2937 * through that code path for re-consideration
2938 * of the validity of that page.
2939 */
2940 must_disconnect = !m->vmp_cs_tainted;
2941 m->vmp_cs_tainted = TRUE;
2942 }
2943 cs_enter_tainted_accepted++;
2944 }
2945 if (kr != KERN_SUCCESS) {
2946 if (cs_debug) {
2947 printf("CODESIGNING: vm_fault_enter(0x%llx): "
2948 "*** INVALID PAGE ***\n",
2949 (long long)vaddr);
2950 }
2951 #if !SECURE_KERNEL
2952 if (cs_enforcement_panic) {
2953 panic("CODESIGNING: panicking on invalid page\n");
2954 }
2955 #endif
2956 }
2957
2958 } else {
2959 /* proceed with the valid page */
2960 kr = KERN_SUCCESS;
2961 }
2962
2963 boolean_t page_queues_locked = FALSE;
2964 #define __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED() \
2965 MACRO_BEGIN \
2966 if (! page_queues_locked) { \
2967 page_queues_locked = TRUE; \
2968 vm_page_lockspin_queues(); \
2969 } \
2970 MACRO_END
2971 #define __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED() \
2972 MACRO_BEGIN \
2973 if (page_queues_locked) { \
2974 page_queues_locked = FALSE; \
2975 vm_page_unlock_queues(); \
2976 } \
2977 MACRO_END
2978
2979 /*
2980 * Hold queues lock to manipulate
2981 * the page queues. Change wiring
2982 * case is obvious.
2983 */
2984 assert((m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) || object != compressor_object);
2985
2986 #if CONFIG_BACKGROUND_QUEUE
2987 vm_page_update_background_state(m);
2988 #endif
2989 if (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
2990 /*
2991 * Compressor pages are neither wired
2992 * nor pageable and should never change.
2993 */
2994 assert(object == compressor_object);
2995 } else if (change_wiring) {
2996 __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED();
2997
2998 if (wired) {
2999 if (kr == KERN_SUCCESS) {
3000 vm_page_wire(m, wire_tag, TRUE);
3001 }
3002 } else {
3003 vm_page_unwire(m, TRUE);
3004 }
3005 /* we keep the page queues lock, if we need it later */
3006
3007 } else {
3008 if (object->internal == TRUE) {
3009 /*
3010 * don't allow anonymous pages on
3011 * the speculative queues
3012 */
3013 no_cache = FALSE;
3014 }
3015 if (kr != KERN_SUCCESS) {
3016 __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED();
3017 vm_page_deactivate(m);
3018 /* we keep the page queues lock, if we need it later */
3019 } else if (((m->vmp_q_state == VM_PAGE_NOT_ON_Q) ||
3020 (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ||
3021 (m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) ||
3022 ((m->vmp_q_state != VM_PAGE_ON_THROTTLED_Q) && no_cache)) &&
3023 !VM_PAGE_WIRED(m)) {
3024
3025 if (vm_page_local_q &&
3026 (*type_of_fault == DBG_COW_FAULT ||
3027 *type_of_fault == DBG_ZERO_FILL_FAULT) ) {
3028 struct vpl *lq;
3029 uint32_t lid;
3030
3031 assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
3032
3033 __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED();
3034 vm_object_lock_assert_exclusive(object);
3035
3036 /*
3037 * we got a local queue to stuff this
3038 * new page on...
3039 * its safe to manipulate local and
3040 * local_id at this point since we're
3041 * behind an exclusive object lock and
3042 * the page is not on any global queue.
3043 *
3044 * we'll use the current cpu number to
3045 * select the queue note that we don't
3046 * need to disable preemption... we're
3047 * going to be behind the local queue's
3048 * lock to do the real work
3049 */
3050 lid = cpu_number();
3051
3052 lq = &vm_page_local_q[lid].vpl_un.vpl;
3053
3054 VPL_LOCK(&lq->vpl_lock);
3055
3056 vm_page_check_pageable_safe(m);
3057 vm_page_queue_enter(&lq->vpl_queue, m,
3058 vm_page_t, vmp_pageq);
3059 m->vmp_q_state = VM_PAGE_ON_ACTIVE_LOCAL_Q;
3060 m->vmp_local_id = lid;
3061 lq->vpl_count++;
3062
3063 if (object->internal)
3064 lq->vpl_internal_count++;
3065 else
3066 lq->vpl_external_count++;
3067
3068 VPL_UNLOCK(&lq->vpl_lock);
3069
3070 if (lq->vpl_count > vm_page_local_q_soft_limit)
3071 {
3072 /*
3073 * we're beyond the soft limit
3074 * for the local queue
3075 * vm_page_reactivate_local will
3076 * 'try' to take the global page
3077 * queue lock... if it can't
3078 * that's ok... we'll let the
3079 * queue continue to grow up
3080 * to the hard limit... at that
3081 * point we'll wait for the
3082 * lock... once we've got the
3083 * lock, we'll transfer all of
3084 * the pages from the local
3085 * queue to the global active
3086 * queue
3087 */
3088 vm_page_reactivate_local(lid, FALSE, FALSE);
3089 }
3090 } else {
3091
3092 __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED();
3093
3094 /*
3095 * test again now that we hold the
3096 * page queue lock
3097 */
3098 if (!VM_PAGE_WIRED(m)) {
3099 if (m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
3100 vm_page_queues_remove(m, FALSE);
3101
3102 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
3103 VM_PAGEOUT_DEBUG(vm_pageout_cleaned_fault_reactivated, 1);
3104 }
3105
3106 if ( !VM_PAGE_ACTIVE_OR_INACTIVE(m) ||
3107 no_cache) {
3108 /*
3109 * If this is a no_cache mapping
3110 * and the page has never been
3111 * mapped before or was
3112 * previously a no_cache page,
3113 * then we want to leave pages
3114 * in the speculative state so
3115 * that they can be readily
3116 * recycled if free memory runs
3117 * low. Otherwise the page is
3118 * activated as normal.
3119 */
3120
3121 if (no_cache &&
3122 (!previously_pmapped ||
3123 m->vmp_no_cache)) {
3124 m->vmp_no_cache = TRUE;
3125
3126 if (m->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q)
3127 vm_page_speculate(m, FALSE);
3128
3129 } else if ( !VM_PAGE_ACTIVE_OR_INACTIVE(m)) {
3130 vm_page_activate(m);
3131 }
3132 }
3133 }
3134 /* we keep the page queues lock, if we need it later */
3135 }
3136 }
3137 }
3138 /* we're done with the page queues lock, if we ever took it */
3139 __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED();
3140
3141
3142 /* If we have a KERN_SUCCESS from the previous checks, we either have
3143 * a good page, or a tainted page that has been accepted by the process.
3144 * In both cases the page will be entered into the pmap.
3145 * If the page is writeable, we need to disconnect it from other pmaps
3146 * now so those processes can take note.
3147 */
3148 if (kr == KERN_SUCCESS) {
3149 /*
3150 * NOTE: we may only hold the vm_object lock SHARED
3151 * at this point, so we need the phys_page lock to
3152 * properly serialize updating the pmapped and
3153 * xpmapped bits
3154 */
3155 if ((prot & VM_PROT_EXECUTE) && !m->vmp_xpmapped) {
3156 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
3157
3158 pmap_lock_phys_page(phys_page);
3159 /*
3160 * go ahead and take the opportunity
3161 * to set 'pmapped' here so that we don't
3162 * need to grab this lock a 2nd time
3163 * just below
3164 */
3165 m->vmp_pmapped = TRUE;
3166
3167 if (!m->vmp_xpmapped) {
3168
3169 m->vmp_xpmapped = TRUE;
3170
3171 pmap_unlock_phys_page(phys_page);
3172
3173 if (!object->internal)
3174 OSAddAtomic(1, &vm_page_xpmapped_external_count);
3175
3176 #if defined(__arm__) || defined(__arm64__)
3177 pmap_sync_page_data_phys(phys_page);
3178 #else
3179 if (object->internal &&
3180 object->pager != NULL) {
3181 /*
3182 * This page could have been
3183 * uncompressed by the
3184 * compressor pager and its
3185 * contents might be only in
3186 * the data cache.
3187 * Since it's being mapped for
3188 * "execute" for the fist time,
3189 * make sure the icache is in
3190 * sync.
3191 */
3192 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
3193 pmap_sync_page_data_phys(phys_page);
3194 }
3195 #endif
3196 } else
3197 pmap_unlock_phys_page(phys_page);
3198 } else {
3199 if (m->vmp_pmapped == FALSE) {
3200 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
3201
3202 pmap_lock_phys_page(phys_page);
3203 m->vmp_pmapped = TRUE;
3204 pmap_unlock_phys_page(phys_page);
3205 }
3206 }
3207
3208 if (fault_type & VM_PROT_WRITE) {
3209
3210 if (m->vmp_wpmapped == FALSE) {
3211 vm_object_lock_assert_exclusive(object);
3212 if (!object->internal && object->pager) {
3213 task_update_logical_writes(current_task(), PAGE_SIZE, TASK_WRITE_DEFERRED, vnode_pager_lookup_vnode(object->pager));
3214 }
3215 m->vmp_wpmapped = TRUE;
3216 }
3217 if (must_disconnect) {
3218 /*
3219 * We can only get here
3220 * because of the CSE logic
3221 */
3222 assert(cs_enforcement_enabled);
3223 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
3224 /*
3225 * If we are faulting for a write, we can clear
3226 * the execute bit - that will ensure the page is
3227 * checked again before being executable, which
3228 * protects against a map switch.
3229 * This only happens the first time the page
3230 * gets tainted, so we won't get stuck here
3231 * to make an already writeable page executable.
3232 */
3233 if (!cs_bypass){
3234 assert(!pmap_has_prot_policy(prot));
3235 prot &= ~VM_PROT_EXECUTE;
3236 }
3237 }
3238 }
3239 assert(VM_PAGE_OBJECT(m) == object);
3240
3241 #if VM_OBJECT_ACCESS_TRACKING
3242 if (object->access_tracking) {
3243 DTRACE_VM2(access_tracking, vm_map_offset_t, vaddr, int, fault_type);
3244 if (fault_type & VM_PROT_WRITE) {
3245 object->access_tracking_writes++;
3246 vm_object_access_tracking_writes++;
3247 } else {
3248 object->access_tracking_reads++;
3249 vm_object_access_tracking_reads++;
3250 }
3251 }
3252 #endif /* VM_OBJECT_ACCESS_TRACKING */
3253
3254 #if PMAP_CS
3255 /*
3256 * If CS enforcement is on, we don't ask for an executable page if the
3257 * fault does not call for execution, because that can fail in
3258 * situations where the caller only actually wanted read access.
3259 * However, it may be better to instead retry without execute on
3260 * failure, or pass a flag into pmap_enter to do the right thing.
3261 */
3262 // TODO: <rdar://problem/30997388> maybe do something better than masking out VM_PROT_EXECUTE on non-execute faults
3263 if (pmap_cs_enforced(pmap) && !(caller_prot & VM_PROT_EXECUTE)) {
3264 prot &= ~VM_PROT_EXECUTE;
3265 }
3266 #endif
3267
3268 /* Prevent a deadlock by not
3269 * holding the object lock if we need to wait for a page in
3270 * pmap_enter() - <rdar://problem/7138958> */
3271 PMAP_ENTER_OPTIONS(pmap, vaddr, m, prot, fault_type, 0,
3272 wired,
3273 pmap_options | PMAP_OPTIONS_NOWAIT,
3274 pe_result);
3275 #if __x86_64__
3276 if (pe_result == KERN_INVALID_ARGUMENT &&
3277 pmap == PMAP_NULL &&
3278 wired) {
3279 /*
3280 * Wiring a page in a pmap-less VM map:
3281 * VMware's "vmmon" kernel extension does this
3282 * to grab pages.
3283 * Let it proceed even though the PMAP_ENTER() failed.
3284 */
3285 pe_result = KERN_SUCCESS;
3286 }
3287 #endif /* __x86_64__ */
3288
3289 if(pe_result == KERN_RESOURCE_SHORTAGE) {
3290
3291 if (need_retry) {
3292 /*
3293 * this will be non-null in the case where we hold the lock
3294 * on the top-object in this chain... we can't just drop
3295 * the lock on the object we're inserting the page into
3296 * and recall the PMAP_ENTER since we can still cause
3297 * a deadlock if one of the critical paths tries to
3298 * acquire the lock on the top-object and we're blocked
3299 * in PMAP_ENTER waiting for memory... our only recourse
3300 * is to deal with it at a higher level where we can
3301 * drop both locks.
3302 */
3303 *need_retry = TRUE;
3304 vm_pmap_enter_retried++;
3305 goto after_the_pmap_enter;
3306 }
3307 /* The nonblocking version of pmap_enter did not succeed.
3308 * and we don't need to drop other locks and retry
3309 * at the level above us, so
3310 * use the blocking version instead. Requires marking
3311 * the page busy and unlocking the object */
3312 boolean_t was_busy = m->vmp_busy;
3313
3314 vm_object_lock_assert_exclusive(object);
3315
3316 m->vmp_busy = TRUE;
3317 vm_object_unlock(object);
3318
3319 PMAP_ENTER_OPTIONS(pmap, vaddr, m, prot, fault_type,
3320 0, wired,
3321 pmap_options, pe_result);
3322
3323 assert(VM_PAGE_OBJECT(m) == object);
3324
3325 /* Take the object lock again. */
3326 vm_object_lock(object);
3327
3328 /* If the page was busy, someone else will wake it up.
3329 * Otherwise, we have to do it now. */
3330 assert(m->vmp_busy);
3331 if(!was_busy) {
3332 PAGE_WAKEUP_DONE(m);
3333 }
3334 vm_pmap_enter_blocked++;
3335 }
3336
3337 kr = pe_result;
3338 }
3339
3340 after_the_pmap_enter:
3341 return kr;
3342 }
3343
3344 void
3345 vm_pre_fault(vm_map_offset_t vaddr)
3346 {
3347 if (pmap_find_phys(current_map()->pmap, vaddr) == 0) {
3348
3349 vm_fault(current_map(), /* map */
3350 vaddr, /* vaddr */
3351 VM_PROT_READ, /* fault_type */
3352 FALSE, /* change_wiring */
3353 VM_KERN_MEMORY_NONE, /* tag - not wiring */
3354 THREAD_UNINT, /* interruptible */
3355 NULL, /* caller_pmap */
3356 0 /* caller_pmap_addr */);
3357 }
3358 }
3359
3360
3361 /*
3362 * Routine: vm_fault
3363 * Purpose:
3364 * Handle page faults, including pseudo-faults
3365 * used to change the wiring status of pages.
3366 * Returns:
3367 * Explicit continuations have been removed.
3368 * Implementation:
3369 * vm_fault and vm_fault_page save mucho state
3370 * in the moral equivalent of a closure. The state
3371 * structure is allocated when first entering vm_fault
3372 * and deallocated when leaving vm_fault.
3373 */
3374
3375 extern int _map_enter_debug;
3376 extern uint64_t get_current_unique_pid(void);
3377
3378 unsigned long vm_fault_collapse_total = 0;
3379 unsigned long vm_fault_collapse_skipped = 0;
3380
3381
3382 kern_return_t
3383 vm_fault_external(
3384 vm_map_t map,
3385 vm_map_offset_t vaddr,
3386 vm_prot_t fault_type,
3387 boolean_t change_wiring,
3388 int interruptible,
3389 pmap_t caller_pmap,
3390 vm_map_offset_t caller_pmap_addr)
3391 {
3392 return vm_fault_internal(map, vaddr, fault_type, change_wiring, vm_tag_bt(),
3393 interruptible, caller_pmap, caller_pmap_addr,
3394 NULL);
3395 }
3396
3397 kern_return_t
3398 vm_fault(
3399 vm_map_t map,
3400 vm_map_offset_t vaddr,
3401 vm_prot_t fault_type,
3402 boolean_t change_wiring,
3403 vm_tag_t wire_tag, /* if wiring must pass tag != VM_KERN_MEMORY_NONE */
3404 int interruptible,
3405 pmap_t caller_pmap,
3406 vm_map_offset_t caller_pmap_addr)
3407 {
3408 return vm_fault_internal(map, vaddr, fault_type, change_wiring, wire_tag,
3409 interruptible, caller_pmap, caller_pmap_addr,
3410 NULL);
3411 }
3412
3413 kern_return_t
3414 vm_fault_internal(
3415 vm_map_t map,
3416 vm_map_offset_t vaddr,
3417 vm_prot_t caller_prot,
3418 boolean_t change_wiring,
3419 vm_tag_t wire_tag, /* if wiring must pass tag != VM_KERN_MEMORY_NONE */
3420 int interruptible,
3421 pmap_t caller_pmap,
3422 vm_map_offset_t caller_pmap_addr,
3423 ppnum_t *physpage_p)
3424 {
3425 vm_map_version_t version; /* Map version for verificiation */
3426 boolean_t wired; /* Should mapping be wired down? */
3427 vm_object_t object; /* Top-level object */
3428 vm_object_offset_t offset; /* Top-level offset */
3429 vm_prot_t prot; /* Protection for mapping */
3430 vm_object_t old_copy_object; /* Saved copy object */
3431 vm_page_t result_page; /* Result of vm_fault_page */
3432 vm_page_t top_page; /* Placeholder page */
3433 kern_return_t kr;
3434
3435 vm_page_t m; /* Fast access to result_page */
3436 kern_return_t error_code;
3437 vm_object_t cur_object;
3438 vm_object_t m_object = NULL;
3439 vm_object_offset_t cur_offset;
3440 vm_page_t cur_m;
3441 vm_object_t new_object;
3442 int type_of_fault;
3443 pmap_t pmap;
3444 wait_interrupt_t interruptible_state;
3445 vm_map_t real_map = map;
3446 vm_map_t original_map = map;
3447 boolean_t object_locks_dropped = FALSE;
3448 vm_prot_t fault_type;
3449 vm_prot_t original_fault_type;
3450 struct vm_object_fault_info fault_info = {};
3451 boolean_t need_collapse = FALSE;
3452 boolean_t need_retry = FALSE;
3453 boolean_t *need_retry_ptr = NULL;
3454 int object_lock_type = 0;
3455 int cur_object_lock_type;
3456 vm_object_t top_object = VM_OBJECT_NULL;
3457 vm_object_t written_on_object = VM_OBJECT_NULL;
3458 memory_object_t written_on_pager = NULL;
3459 vm_object_offset_t written_on_offset = 0;
3460 int throttle_delay;
3461 int compressed_count_delta;
3462 int grab_options;
3463 vm_map_offset_t trace_vaddr;
3464 vm_map_offset_t trace_real_vaddr;
3465 #if DEVELOPMENT || DEBUG
3466 vm_map_offset_t real_vaddr;
3467
3468 real_vaddr = vaddr;
3469 #endif /* DEVELOPMENT || DEBUG */
3470 trace_real_vaddr = vaddr;
3471 vaddr = vm_map_trunc_page(vaddr, PAGE_MASK);
3472
3473 if (map == kernel_map) {
3474 trace_vaddr = VM_KERNEL_ADDRHIDE(vaddr);
3475 trace_real_vaddr = VM_KERNEL_ADDRHIDE(trace_real_vaddr);
3476 } else {
3477 trace_vaddr = vaddr;
3478 }
3479
3480 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
3481 (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_START,
3482 ((uint64_t)trace_vaddr >> 32),
3483 trace_vaddr,
3484 (map == kernel_map),
3485 0,
3486 0);
3487
3488 if (get_preemption_level() != 0) {
3489 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
3490 (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END,
3491 ((uint64_t)trace_vaddr >> 32),
3492 trace_vaddr,
3493 KERN_FAILURE,
3494 0,
3495 0);
3496
3497 return (KERN_FAILURE);
3498 }
3499
3500 thread_t cthread = current_thread();
3501 boolean_t rtfault = (cthread->sched_mode == TH_MODE_REALTIME);
3502 uint64_t fstart = 0;
3503
3504 if (rtfault) {
3505 fstart = mach_continuous_time();
3506 }
3507
3508 interruptible_state = thread_interrupt_level(interruptible);
3509
3510 fault_type = (change_wiring ? VM_PROT_NONE : caller_prot);
3511
3512 VM_STAT_INCR(faults);
3513 current_task()->faults++;
3514 original_fault_type = fault_type;
3515
3516 if (fault_type & VM_PROT_WRITE)
3517 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3518 else
3519 object_lock_type = OBJECT_LOCK_SHARED;
3520
3521 cur_object_lock_type = OBJECT_LOCK_SHARED;
3522
3523 if ((map == kernel_map) && (caller_prot & VM_PROT_WRITE)) {
3524 if (compressor_map) {
3525 if ((vaddr >= vm_map_min(compressor_map)) && (vaddr < vm_map_max(compressor_map))) {
3526 panic("Write fault on compressor map, va: %p type: %u bounds: %p->%p", (void *) vaddr, caller_prot, (void *) vm_map_min(compressor_map), (void *) vm_map_max(compressor_map));
3527
3528 }
3529 }
3530 }
3531 RetryFault:
3532 assert(written_on_object == VM_OBJECT_NULL);
3533
3534 /*
3535 * assume we will hit a page in the cache
3536 * otherwise, explicitly override with
3537 * the real fault type once we determine it
3538 */
3539 type_of_fault = DBG_CACHE_HIT_FAULT;
3540
3541 /*
3542 * Find the backing store object and offset into
3543 * it to begin the search.
3544 */
3545 fault_type = original_fault_type;
3546 map = original_map;
3547 vm_map_lock_read(map);
3548
3549 kr = vm_map_lookup_locked(&map, vaddr, fault_type,
3550 object_lock_type, &version,
3551 &object, &offset, &prot, &wired,
3552 &fault_info,
3553 &real_map);
3554
3555 if (kr != KERN_SUCCESS) {
3556 vm_map_unlock_read(map);
3557 goto done;
3558 }
3559 pmap = real_map->pmap;
3560 fault_info.interruptible = interruptible;
3561 fault_info.stealth = FALSE;
3562 fault_info.io_sync = FALSE;
3563 fault_info.mark_zf_absent = FALSE;
3564 fault_info.batch_pmap_op = FALSE;
3565
3566 /*
3567 * If the page is wired, we must fault for the current protection
3568 * value, to avoid further faults.
3569 */
3570 if (wired) {
3571 fault_type = prot | VM_PROT_WRITE;
3572 /*
3573 * since we're treating this fault as a 'write'
3574 * we must hold the top object lock exclusively
3575 */
3576 if (object_lock_type == OBJECT_LOCK_SHARED) {
3577
3578 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3579
3580 if (vm_object_lock_upgrade(object) == FALSE) {
3581 /*
3582 * couldn't upgrade, so explictly
3583 * take the lock exclusively
3584 */
3585 vm_object_lock(object);
3586 }
3587 }
3588 }
3589
3590 #if VM_FAULT_CLASSIFY
3591 /*
3592 * Temporary data gathering code
3593 */
3594 vm_fault_classify(object, offset, fault_type);
3595 #endif
3596 /*
3597 * Fast fault code. The basic idea is to do as much as
3598 * possible while holding the map lock and object locks.
3599 * Busy pages are not used until the object lock has to
3600 * be dropped to do something (copy, zero fill, pmap enter).
3601 * Similarly, paging references aren't acquired until that
3602 * point, and object references aren't used.
3603 *
3604 * If we can figure out what to do
3605 * (zero fill, copy on write, pmap enter) while holding
3606 * the locks, then it gets done. Otherwise, we give up,
3607 * and use the original fault path (which doesn't hold
3608 * the map lock, and relies on busy pages).
3609 * The give up cases include:
3610 * - Have to talk to pager.
3611 * - Page is busy, absent or in error.
3612 * - Pager has locked out desired access.
3613 * - Fault needs to be restarted.
3614 * - Have to push page into copy object.
3615 *
3616 * The code is an infinite loop that moves one level down
3617 * the shadow chain each time. cur_object and cur_offset
3618 * refer to the current object being examined. object and offset
3619 * are the original object from the map. The loop is at the
3620 * top level if and only if object and cur_object are the same.
3621 *
3622 * Invariants: Map lock is held throughout. Lock is held on
3623 * original object and cur_object (if different) when
3624 * continuing or exiting loop.
3625 *
3626 */
3627
3628 #if defined(__arm64__)
3629 /*
3630 * Fail if reading an execute-only page in a
3631 * pmap that enforces execute-only protection.
3632 */
3633 if (fault_type == VM_PROT_READ &&
3634 (prot & VM_PROT_EXECUTE) &&
3635 !(prot & VM_PROT_READ) &&
3636 pmap_enforces_execute_only(pmap)) {
3637 vm_object_unlock(object);
3638 vm_map_unlock_read(map);
3639 if (real_map != map) {
3640 vm_map_unlock(real_map);
3641 }
3642 kr = KERN_PROTECTION_FAILURE;
3643 goto done;
3644 }
3645 #endif
3646
3647 /*
3648 * If this page is to be inserted in a copy delay object
3649 * for writing, and if the object has a copy, then the
3650 * copy delay strategy is implemented in the slow fault page.
3651 */
3652 if (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY &&
3653 object->copy != VM_OBJECT_NULL && (fault_type & VM_PROT_WRITE))
3654 goto handle_copy_delay;
3655
3656 cur_object = object;
3657 cur_offset = offset;
3658
3659 grab_options = 0;
3660 #if CONFIG_SECLUDED_MEMORY
3661 if (object->can_grab_secluded) {
3662 grab_options |= VM_PAGE_GRAB_SECLUDED;
3663 }
3664 #endif /* CONFIG_SECLUDED_MEMORY */
3665
3666 while (TRUE) {
3667 if (!cur_object->pager_created &&
3668 cur_object->phys_contiguous) /* superpage */
3669 break;
3670
3671 if (cur_object->blocked_access) {
3672 /*
3673 * Access to this VM object has been blocked.
3674 * Let the slow path handle it.
3675 */
3676 break;
3677 }
3678
3679 m = vm_page_lookup(cur_object, cur_offset);
3680 m_object = NULL;
3681
3682 if (m != VM_PAGE_NULL) {
3683 m_object = cur_object;
3684
3685 if (m->vmp_busy) {
3686 wait_result_t result;
3687
3688 /*
3689 * in order to do the PAGE_ASSERT_WAIT, we must
3690 * have object that 'm' belongs to locked exclusively
3691 */
3692 if (object != cur_object) {
3693
3694 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3695
3696 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3697
3698 if (vm_object_lock_upgrade(cur_object) == FALSE) {
3699 /*
3700 * couldn't upgrade so go do a full retry
3701 * immediately since we can no longer be
3702 * certain about cur_object (since we
3703 * don't hold a reference on it)...
3704 * first drop the top object lock
3705 */
3706 vm_object_unlock(object);
3707
3708 vm_map_unlock_read(map);
3709 if (real_map != map)
3710 vm_map_unlock(real_map);
3711
3712 goto RetryFault;
3713 }
3714 }
3715 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3716
3717 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3718
3719 if (vm_object_lock_upgrade(object) == FALSE) {
3720 /*
3721 * couldn't upgrade, so explictly take the lock
3722 * exclusively and go relookup the page since we
3723 * will have dropped the object lock and
3724 * a different thread could have inserted
3725 * a page at this offset
3726 * no need for a full retry since we're
3727 * at the top level of the object chain
3728 */
3729 vm_object_lock(object);
3730
3731 continue;
3732 }
3733 }
3734 if ((m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) && m_object->internal) {
3735 /*
3736 * m->vmp_busy == TRUE and the object is locked exclusively
3737 * if m->pageout_queue == TRUE after we acquire the
3738 * queues lock, we are guaranteed that it is stable on
3739 * the pageout queue and therefore reclaimable
3740 *
3741 * NOTE: this is only true for the internal pageout queue
3742 * in the compressor world
3743 */
3744 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
3745
3746 vm_page_lock_queues();
3747
3748 if (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
3749 vm_pageout_throttle_up(m);
3750 vm_page_unlock_queues();
3751
3752 PAGE_WAKEUP_DONE(m);
3753 goto reclaimed_from_pageout;
3754 }
3755 vm_page_unlock_queues();
3756 }
3757 if (object != cur_object)
3758 vm_object_unlock(object);
3759
3760 vm_map_unlock_read(map);
3761 if (real_map != map)
3762 vm_map_unlock(real_map);
3763
3764 result = PAGE_ASSERT_WAIT(m, interruptible);
3765
3766 vm_object_unlock(cur_object);
3767
3768 if (result == THREAD_WAITING) {
3769 result = thread_block(THREAD_CONTINUE_NULL);
3770
3771 counter(c_vm_fault_page_block_busy_kernel++);
3772 }
3773 if (result == THREAD_AWAKENED || result == THREAD_RESTART)
3774 goto RetryFault;
3775
3776 kr = KERN_ABORTED;
3777 goto done;
3778 }
3779 reclaimed_from_pageout:
3780 if (m->vmp_laundry) {
3781 if (object != cur_object) {
3782 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3783 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3784
3785 vm_object_unlock(object);
3786 vm_object_unlock(cur_object);
3787
3788 vm_map_unlock_read(map);
3789 if (real_map != map)
3790 vm_map_unlock(real_map);
3791
3792 goto RetryFault;
3793 }
3794
3795 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3796
3797 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3798
3799 if (vm_object_lock_upgrade(object) == FALSE) {
3800 /*
3801 * couldn't upgrade, so explictly take the lock
3802 * exclusively and go relookup the page since we
3803 * will have dropped the object lock and
3804 * a different thread could have inserted
3805 * a page at this offset
3806 * no need for a full retry since we're
3807 * at the top level of the object chain
3808 */
3809 vm_object_lock(object);
3810
3811 continue;
3812 }
3813 }
3814 vm_pageout_steal_laundry(m, FALSE);
3815 }
3816
3817 if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
3818 /*
3819 * Guard page: let the slow path deal with it
3820 */
3821 break;
3822 }
3823 if (m->vmp_unusual && (m->vmp_error || m->vmp_restart || m->vmp_private || m->vmp_absent)) {
3824 /*
3825 * Unusual case... let the slow path deal with it
3826 */
3827 break;
3828 }
3829 if (VM_OBJECT_PURGEABLE_FAULT_ERROR(m_object)) {
3830 if (object != cur_object)
3831 vm_object_unlock(object);
3832 vm_map_unlock_read(map);
3833 if (real_map != map)
3834 vm_map_unlock(real_map);
3835 vm_object_unlock(cur_object);
3836 kr = KERN_MEMORY_ERROR;
3837 goto done;
3838 }
3839 assert(m_object == VM_PAGE_OBJECT(m));
3840
3841 if (VM_FAULT_NEED_CS_VALIDATION(map->pmap, m, m_object) ||
3842 (physpage_p != NULL && (prot & VM_PROT_WRITE))) {
3843 upgrade_for_validation:
3844 /*
3845 * We might need to validate this page
3846 * against its code signature, so we
3847 * want to hold the VM object exclusively.
3848 */
3849 if (object != cur_object) {
3850 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3851 vm_object_unlock(object);
3852 vm_object_unlock(cur_object);
3853
3854 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3855
3856 vm_map_unlock_read(map);
3857 if (real_map != map)
3858 vm_map_unlock(real_map);
3859
3860 goto RetryFault;
3861 }
3862
3863 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3864
3865 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3866
3867 if (vm_object_lock_upgrade(object) == FALSE) {
3868 /*
3869 * couldn't upgrade, so explictly take the lock
3870 * exclusively and go relookup the page since we
3871 * will have dropped the object lock and
3872 * a different thread could have inserted
3873 * a page at this offset
3874 * no need for a full retry since we're
3875 * at the top level of the object chain
3876 */
3877 vm_object_lock(object);
3878
3879 continue;
3880 }
3881 }
3882 }
3883 /*
3884 * Two cases of map in faults:
3885 * - At top level w/o copy object.
3886 * - Read fault anywhere.
3887 * --> must disallow write.
3888 */
3889
3890 if (object == cur_object && object->copy == VM_OBJECT_NULL) {
3891
3892 goto FastPmapEnter;
3893 }
3894
3895 if ((fault_type & VM_PROT_WRITE) == 0) {
3896 if (!pmap_has_prot_policy(prot)) {
3897 prot &= ~VM_PROT_WRITE;
3898 } else {
3899 /*
3900 * For a protection that the pmap cares
3901 * about, we must hand over the full
3902 * set of protections (so that the pmap
3903 * layer can apply any desired policy).
3904 * This means that cs_bypass must be
3905 * set, as this can force us to pass
3906 * RWX.
3907 */
3908 assert(fault_info.cs_bypass);
3909 }
3910
3911 if (object != cur_object) {
3912 /*
3913 * We still need to hold the top object
3914 * lock here to prevent a race between
3915 * a read fault (taking only "shared"
3916 * locks) and a write fault (taking
3917 * an "exclusive" lock on the top
3918 * object.
3919 * Otherwise, as soon as we release the
3920 * top lock, the write fault could
3921 * proceed and actually complete before
3922 * the read fault, and the copied page's
3923 * translation could then be overwritten
3924 * by the read fault's translation for
3925 * the original page.
3926 *
3927 * Let's just record what the top object
3928 * is and we'll release it later.
3929 */
3930 top_object = object;
3931
3932 /*
3933 * switch to the object that has the new page
3934 */
3935 object = cur_object;
3936 object_lock_type = cur_object_lock_type;
3937 }
3938 FastPmapEnter:
3939 assert(m_object == VM_PAGE_OBJECT(m));
3940
3941 /*
3942 * prepare for the pmap_enter...
3943 * object and map are both locked
3944 * m contains valid data
3945 * object == m->vmp_object
3946 * cur_object == NULL or it's been unlocked
3947 * no paging references on either object or cur_object
3948 */
3949 if (top_object != VM_OBJECT_NULL || object_lock_type != OBJECT_LOCK_EXCLUSIVE)
3950 need_retry_ptr = &need_retry;
3951 else
3952 need_retry_ptr = NULL;
3953
3954 if (caller_pmap) {
3955 kr = vm_fault_enter(m,
3956 caller_pmap,
3957 caller_pmap_addr,
3958 prot,
3959 caller_prot,
3960 wired,
3961 change_wiring,
3962 wire_tag,
3963 &fault_info,
3964 need_retry_ptr,
3965 &type_of_fault);
3966 } else {
3967 kr = vm_fault_enter(m,
3968 pmap,
3969 vaddr,
3970 prot,
3971 caller_prot,
3972 wired,
3973 change_wiring,
3974 wire_tag,
3975 &fault_info,
3976 need_retry_ptr,
3977 &type_of_fault);
3978 }
3979 #if DEVELOPMENT || DEBUG
3980 {
3981 int event_code = 0;
3982
3983 if (m_object->internal)
3984 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_INTERNAL));
3985 else if (m_object->object_is_shared_cache)
3986 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_SHAREDCACHE));
3987 else
3988 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_EXTERNAL));
3989
3990 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, event_code, trace_real_vaddr, (fault_info.user_tag << 16) | (caller_prot << 8) | type_of_fault, m->vmp_offset, get_current_unique_pid(), 0);
3991
3992 DTRACE_VM6(real_fault, vm_map_offset_t, real_vaddr, vm_map_offset_t, m->vmp_offset, int, event_code, int, caller_prot, int, type_of_fault, int, fault_info.user_tag);
3993 }
3994 #endif
3995 if (kr == KERN_SUCCESS &&
3996 physpage_p != NULL) {
3997 /* for vm_map_wire_and_extract() */
3998 *physpage_p = VM_PAGE_GET_PHYS_PAGE(m);
3999 if (prot & VM_PROT_WRITE) {
4000 vm_object_lock_assert_exclusive(m_object);
4001 m->vmp_dirty = TRUE;
4002 }
4003 }
4004
4005 if (top_object != VM_OBJECT_NULL) {
4006 /*
4007 * It's safe to drop the top object
4008 * now that we've done our
4009 * vm_fault_enter(). Any other fault
4010 * in progress for that virtual
4011 * address will either find our page
4012 * and translation or put in a new page
4013 * and translation.
4014 */
4015 vm_object_unlock(top_object);
4016 top_object = VM_OBJECT_NULL;
4017 }
4018
4019 if (need_collapse == TRUE)
4020 vm_object_collapse(object, offset, TRUE);
4021
4022 if (need_retry == FALSE &&
4023 (type_of_fault == DBG_PAGEIND_FAULT || type_of_fault == DBG_PAGEINV_FAULT || type_of_fault == DBG_CACHE_HIT_FAULT)) {
4024 /*
4025 * evaluate access pattern and update state
4026 * vm_fault_deactivate_behind depends on the
4027 * state being up to date
4028 */
4029 vm_fault_is_sequential(m_object, cur_offset, fault_info.behavior);
4030
4031 vm_fault_deactivate_behind(m_object, cur_offset, fault_info.behavior);
4032 }
4033 /*
4034 * That's it, clean up and return.
4035 */
4036 if (m->vmp_busy)
4037 PAGE_WAKEUP_DONE(m);
4038
4039 if (need_retry == FALSE && !m_object->internal && (fault_type & VM_PROT_WRITE)) {
4040
4041 vm_object_paging_begin(m_object);
4042
4043 assert(written_on_object == VM_OBJECT_NULL);
4044 written_on_object = m_object;
4045 written_on_pager = m_object->pager;
4046 written_on_offset = m_object->paging_offset + m->vmp_offset;
4047 }
4048 vm_object_unlock(object);
4049
4050 vm_map_unlock_read(map);
4051 if (real_map != map)
4052 vm_map_unlock(real_map);
4053
4054 if (need_retry == TRUE) {
4055 /*
4056 * vm_fault_enter couldn't complete the PMAP_ENTER...
4057 * at this point we don't hold any locks so it's safe
4058 * to ask the pmap layer to expand the page table to
4059 * accommodate this mapping... once expanded, we'll
4060 * re-drive the fault which should result in vm_fault_enter
4061 * being able to successfully enter the mapping this time around
4062 */
4063 (void)pmap_enter_options(
4064 pmap, vaddr, 0, 0, 0, 0, 0,
4065 PMAP_OPTIONS_NOENTER, NULL);
4066
4067 need_retry = FALSE;
4068 goto RetryFault;
4069 }
4070 goto done;
4071 }
4072 /*
4073 * COPY ON WRITE FAULT
4074 */
4075 assert(object_lock_type == OBJECT_LOCK_EXCLUSIVE);
4076
4077 /*
4078 * If objects match, then
4079 * object->copy must not be NULL (else control
4080 * would be in previous code block), and we
4081 * have a potential push into the copy object
4082 * with which we can't cope with here.
4083 */
4084 if (cur_object == object) {
4085 /*
4086 * must take the slow path to
4087 * deal with the copy push
4088 */
4089 break;
4090 }
4091
4092 /*
4093 * This is now a shadow based copy on write
4094 * fault -- it requires a copy up the shadow
4095 * chain.
4096 */
4097 assert(m_object == VM_PAGE_OBJECT(m));
4098
4099 if ((cur_object_lock_type == OBJECT_LOCK_SHARED) &&
4100 VM_FAULT_NEED_CS_VALIDATION(NULL, m, m_object)) {
4101 goto upgrade_for_validation;
4102 }
4103
4104 /*
4105 * Allocate a page in the original top level
4106 * object. Give up if allocate fails. Also
4107 * need to remember current page, as it's the
4108 * source of the copy.
4109 *
4110 * at this point we hold locks on both
4111 * object and cur_object... no need to take
4112 * paging refs or mark pages BUSY since
4113 * we don't drop either object lock until
4114 * the page has been copied and inserted
4115 */
4116 cur_m = m;
4117 m = vm_page_grab_options(grab_options);
4118 m_object = NULL;
4119
4120 if (m == VM_PAGE_NULL) {
4121 /*
4122 * no free page currently available...
4123 * must take the slow path
4124 */
4125 break;
4126 }
4127 /*
4128 * Now do the copy. Mark the source page busy...
4129 *
4130 * NOTE: This code holds the map lock across
4131 * the page copy.
4132 */
4133 vm_page_copy(cur_m, m);
4134 vm_page_insert(m, object, offset);
4135 m_object = object;
4136 SET_PAGE_DIRTY(m, FALSE);
4137
4138 /*
4139 * Now cope with the source page and object
4140 */
4141 if (object->ref_count > 1 && cur_m->vmp_pmapped)
4142 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(cur_m));
4143
4144 if (cur_m->vmp_clustered) {
4145 VM_PAGE_COUNT_AS_PAGEIN(cur_m);
4146 VM_PAGE_CONSUME_CLUSTERED(cur_m);
4147 vm_fault_is_sequential(cur_object, cur_offset, fault_info.behavior);
4148 }
4149 need_collapse = TRUE;
4150
4151 if (!cur_object->internal &&
4152 cur_object->copy_strategy == MEMORY_OBJECT_COPY_DELAY) {
4153 /*
4154 * The object from which we've just
4155 * copied a page is most probably backed
4156 * by a vnode. We don't want to waste too
4157 * much time trying to collapse the VM objects
4158 * and create a bottleneck when several tasks
4159 * map the same file.
4160 */
4161 if (cur_object->copy == object) {
4162 /*
4163 * Shared mapping or no COW yet.
4164 * We can never collapse a copy
4165 * object into its backing object.
4166 */
4167 need_collapse = FALSE;
4168 } else if (cur_object->copy == object->shadow &&
4169 object->shadow->resident_page_count == 0) {
4170 /*
4171 * Shared mapping after a COW occurred.
4172 */
4173 need_collapse = FALSE;
4174 }
4175 }
4176 vm_object_unlock(cur_object);
4177
4178 if (need_collapse == FALSE)
4179 vm_fault_collapse_skipped++;
4180 vm_fault_collapse_total++;
4181
4182 type_of_fault = DBG_COW_FAULT;
4183 VM_STAT_INCR(cow_faults);
4184 DTRACE_VM2(cow_fault, int, 1, (uint64_t *), NULL);
4185 current_task()->cow_faults++;
4186
4187 goto FastPmapEnter;
4188
4189 } else {
4190 /*
4191 * No page at cur_object, cur_offset... m == NULL
4192 */
4193 if (cur_object->pager_created) {
4194 int compressor_external_state = VM_EXTERNAL_STATE_UNKNOWN;
4195
4196 if (MUST_ASK_PAGER(cur_object, cur_offset, compressor_external_state) == TRUE) {
4197 int my_fault_type;
4198 int c_flags = C_DONT_BLOCK;
4199 boolean_t insert_cur_object = FALSE;
4200
4201 /*
4202 * May have to talk to a pager...
4203 * if so, take the slow path by
4204 * doing a 'break' from the while (TRUE) loop
4205 *
4206 * external_state will only be set to VM_EXTERNAL_STATE_EXISTS
4207 * if the compressor is active and the page exists there
4208 */
4209 if (compressor_external_state != VM_EXTERNAL_STATE_EXISTS)
4210 break;
4211
4212 if (map == kernel_map || real_map == kernel_map) {
4213 /*
4214 * can't call into the compressor with the kernel_map
4215 * lock held, since the compressor may try to operate
4216 * on the kernel map in order to return an empty c_segment
4217 */
4218 break;
4219 }
4220 if (object != cur_object) {
4221 if (fault_type & VM_PROT_WRITE)
4222 c_flags |= C_KEEP;
4223 else
4224 insert_cur_object = TRUE;
4225 }
4226 if (insert_cur_object == TRUE) {
4227
4228 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
4229
4230 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4231
4232 if (vm_object_lock_upgrade(cur_object) == FALSE) {
4233 /*
4234 * couldn't upgrade so go do a full retry
4235 * immediately since we can no longer be
4236 * certain about cur_object (since we
4237 * don't hold a reference on it)...
4238 * first drop the top object lock
4239 */
4240 vm_object_unlock(object);
4241
4242 vm_map_unlock_read(map);
4243 if (real_map != map)
4244 vm_map_unlock(real_map);
4245
4246 goto RetryFault;
4247 }
4248 }
4249 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
4250
4251 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4252
4253 if (object != cur_object) {
4254 /*
4255 * we can't go for the upgrade on the top
4256 * lock since the upgrade may block waiting
4257 * for readers to drain... since we hold
4258 * cur_object locked at this point, waiting
4259 * for the readers to drain would represent
4260 * a lock order inversion since the lock order
4261 * for objects is the reference order in the
4262 * shadown chain
4263 */
4264 vm_object_unlock(object);
4265 vm_object_unlock(cur_object);
4266
4267 vm_map_unlock_read(map);
4268 if (real_map != map)
4269 vm_map_unlock(real_map);
4270
4271 goto RetryFault;
4272 }
4273 if (vm_object_lock_upgrade(object) == FALSE) {
4274 /*
4275 * couldn't upgrade, so explictly take the lock
4276 * exclusively and go relookup the page since we
4277 * will have dropped the object lock and
4278 * a different thread could have inserted
4279 * a page at this offset
4280 * no need for a full retry since we're
4281 * at the top level of the object chain
4282 */
4283 vm_object_lock(object);
4284
4285 continue;
4286 }
4287 }
4288 m = vm_page_grab_options(grab_options);
4289 m_object = NULL;
4290
4291 if (m == VM_PAGE_NULL) {
4292 /*
4293 * no free page currently available...
4294 * must take the slow path
4295 */
4296 break;
4297 }
4298
4299 /*
4300 * The object is and remains locked
4301 * so no need to take a
4302 * "paging_in_progress" reference.
4303 */
4304 boolean_t shared_lock;
4305 if ((object == cur_object &&
4306 object_lock_type == OBJECT_LOCK_EXCLUSIVE) ||
4307 (object != cur_object &&
4308 cur_object_lock_type == OBJECT_LOCK_EXCLUSIVE)) {
4309 shared_lock = FALSE;
4310 } else {
4311 shared_lock = TRUE;
4312 }
4313
4314 kr = vm_compressor_pager_get(
4315 cur_object->pager,
4316 (cur_offset +
4317 cur_object->paging_offset),
4318 VM_PAGE_GET_PHYS_PAGE(m),
4319 &my_fault_type,
4320 c_flags,
4321 &compressed_count_delta);
4322
4323 vm_compressor_pager_count(
4324 cur_object->pager,
4325 compressed_count_delta,
4326 shared_lock,
4327 cur_object);
4328
4329 if (kr != KERN_SUCCESS) {
4330 vm_page_release(m, FALSE);
4331 m = VM_PAGE_NULL;
4332 break;
4333 }
4334 m->vmp_dirty = TRUE;
4335
4336 /*
4337 * If the object is purgeable, its
4338 * owner's purgeable ledgers will be
4339 * updated in vm_page_insert() but the
4340 * page was also accounted for in a
4341 * "compressed purgeable" ledger, so
4342 * update that now.
4343 */
4344 if (object != cur_object &&
4345 !insert_cur_object) {
4346 /*
4347 * We're not going to insert
4348 * the decompressed page into
4349 * the object it came from.
4350 *
4351 * We're dealing with a
4352 * copy-on-write fault on
4353 * "object".
4354 * We're going to decompress
4355 * the page directly into the
4356 * target "object" while
4357 * keepin the compressed
4358 * page for "cur_object", so
4359 * no ledger update in that
4360 * case.
4361 */
4362 } else if (((cur_object->purgable ==
4363 VM_PURGABLE_DENY) &&
4364 (!cur_object->vo_ledger_tag)) ||
4365 (cur_object->vo_owner ==
4366 NULL)) {
4367 /*
4368 * "cur_object" is not purgeable
4369 * and is not ledger-taged, or
4370 * there's no owner for it,
4371 * so no owner's ledgers to
4372 * update.
4373 */
4374 } else {
4375 /*
4376 * One less compressed
4377 * purgeable/tagged page for
4378 * cur_object's owner.
4379 */
4380 vm_object_owner_compressed_update(
4381 cur_object,
4382 -1);
4383 }
4384
4385 if (insert_cur_object) {
4386 vm_page_insert(m, cur_object, cur_offset);
4387 m_object = cur_object;
4388 } else {
4389 vm_page_insert(m, object, offset);
4390 m_object = object;
4391 }
4392
4393 if ((m_object->wimg_bits & VM_WIMG_MASK) != VM_WIMG_USE_DEFAULT) {
4394 /*
4395 * If the page is not cacheable,
4396 * we can't let its contents
4397 * linger in the data cache
4398 * after the decompression.
4399 */
4400 pmap_sync_page_attributes_phys(VM_PAGE_GET_PHYS_PAGE(m));
4401 }
4402
4403 type_of_fault = my_fault_type;
4404
4405 VM_STAT_INCR(decompressions);
4406
4407 if (cur_object != object) {
4408 if (insert_cur_object) {
4409 top_object = object;
4410 /*
4411 * switch to the object that has the new page
4412 */
4413 object = cur_object;
4414 object_lock_type = cur_object_lock_type;
4415 } else {
4416 vm_object_unlock(cur_object);
4417 cur_object = object;
4418 }
4419 }
4420 goto FastPmapEnter;
4421 }
4422 /*
4423 * existence map present and indicates
4424 * that the pager doesn't have this page
4425 */
4426 }
4427 if (cur_object->shadow == VM_OBJECT_NULL) {
4428 /*
4429 * Zero fill fault. Page gets
4430 * inserted into the original object.
4431 */
4432 if (cur_object->shadow_severed ||
4433 VM_OBJECT_PURGEABLE_FAULT_ERROR(cur_object) ||
4434 cur_object == compressor_object ||
4435 cur_object == kernel_object ||
4436 cur_object == vm_submap_object) {
4437 if (object != cur_object)
4438 vm_object_unlock(cur_object);
4439 vm_object_unlock(object);
4440
4441 vm_map_unlock_read(map);
4442 if (real_map != map)
4443 vm_map_unlock(real_map);
4444
4445 kr = KERN_MEMORY_ERROR;
4446 goto done;
4447 }
4448 if (cur_object != object) {
4449 vm_object_unlock(cur_object);
4450
4451 cur_object = object;
4452 }
4453 if (object_lock_type == OBJECT_LOCK_SHARED) {
4454
4455 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4456
4457 if (vm_object_lock_upgrade(object) == FALSE) {
4458 /*
4459 * couldn't upgrade so do a full retry on the fault
4460 * since we dropped the object lock which
4461 * could allow another thread to insert
4462 * a page at this offset
4463 */
4464 vm_map_unlock_read(map);
4465 if (real_map != map)
4466 vm_map_unlock(real_map);
4467
4468 goto RetryFault;
4469 }
4470 }
4471 m = vm_page_alloc(object, offset);
4472 m_object = NULL;
4473
4474 if (m == VM_PAGE_NULL) {
4475 /*
4476 * no free page currently available...
4477 * must take the slow path
4478 */
4479 break;
4480 }
4481 m_object = object;
4482
4483 /*
4484 * Now zero fill page...
4485 * the page is probably going to
4486 * be written soon, so don't bother
4487 * to clear the modified bit
4488 *
4489 * NOTE: This code holds the map
4490 * lock across the zero fill.
4491 */
4492 type_of_fault = vm_fault_zero_page(m, map->no_zero_fill);
4493
4494 goto FastPmapEnter;
4495 }
4496 /*
4497 * On to the next level in the shadow chain
4498 */
4499 cur_offset += cur_object->vo_shadow_offset;
4500 new_object = cur_object->shadow;
4501
4502 /*
4503 * take the new_object's lock with the indicated state
4504 */
4505 if (cur_object_lock_type == OBJECT_LOCK_SHARED)
4506 vm_object_lock_shared(new_object);
4507 else
4508 vm_object_lock(new_object);
4509
4510 if (cur_object != object)
4511 vm_object_unlock(cur_object);
4512
4513 cur_object = new_object;
4514
4515 continue;
4516 }
4517 }
4518 /*
4519 * Cleanup from fast fault failure. Drop any object
4520 * lock other than original and drop map lock.
4521 */
4522 if (object != cur_object)
4523 vm_object_unlock(cur_object);
4524
4525 /*
4526 * must own the object lock exclusively at this point
4527 */
4528 if (object_lock_type == OBJECT_LOCK_SHARED) {
4529 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4530
4531 if (vm_object_lock_upgrade(object) == FALSE) {
4532 /*
4533 * couldn't upgrade, so explictly
4534 * take the lock exclusively
4535 * no need to retry the fault at this
4536 * point since "vm_fault_page" will
4537 * completely re-evaluate the state
4538 */
4539 vm_object_lock(object);
4540 }
4541 }
4542
4543 handle_copy_delay:
4544 vm_map_unlock_read(map);
4545 if (real_map != map)
4546 vm_map_unlock(real_map);
4547
4548 if (__improbable(object == compressor_object ||
4549 object == kernel_object ||
4550 object == vm_submap_object)) {
4551 /*
4552 * These objects are explicitly managed and populated by the
4553 * kernel. The virtual ranges backed by these objects should
4554 * either have wired pages or "holes" that are not supposed to
4555 * be accessed at all until they get explicitly populated.
4556 * We should never have to resolve a fault on a mapping backed
4557 * by one of these VM objects and providing a zero-filled page
4558 * would be wrong here, so let's fail the fault and let the
4559 * caller crash or recover.
4560 */
4561 vm_object_unlock(object);
4562 kr = KERN_MEMORY_ERROR;
4563 goto done;
4564 }
4565
4566 assert(object != compressor_object);
4567 assert(object != kernel_object);
4568 assert(object != vm_submap_object);
4569
4570 /*
4571 * Make a reference to this object to
4572 * prevent its disposal while we are messing with
4573 * it. Once we have the reference, the map is free
4574 * to be diddled. Since objects reference their
4575 * shadows (and copies), they will stay around as well.
4576 */
4577 vm_object_reference_locked(object);
4578 vm_object_paging_begin(object);
4579
4580 XPR(XPR_VM_FAULT,"vm_fault -> vm_fault_page\n",0,0,0,0,0);
4581
4582 error_code = 0;
4583
4584 result_page = VM_PAGE_NULL;
4585 kr = vm_fault_page(object, offset, fault_type,
4586 (change_wiring && !wired),
4587 FALSE, /* page not looked up */
4588 &prot, &result_page, &top_page,
4589 &type_of_fault,
4590 &error_code, map->no_zero_fill,
4591 FALSE, &fault_info);
4592
4593 /*
4594 * if kr != VM_FAULT_SUCCESS, then the paging reference
4595 * has been dropped and the object unlocked... the ref_count
4596 * is still held
4597 *
4598 * if kr == VM_FAULT_SUCCESS, then the paging reference
4599 * is still held along with the ref_count on the original object
4600 *
4601 * the object is returned locked with a paging reference
4602 *
4603 * if top_page != NULL, then it's BUSY and the
4604 * object it belongs to has a paging reference
4605 * but is returned unlocked
4606 */
4607 if (kr != VM_FAULT_SUCCESS &&
4608 kr != VM_FAULT_SUCCESS_NO_VM_PAGE) {
4609 /*
4610 * we didn't succeed, lose the object reference immediately.
4611 */
4612 vm_object_deallocate(object);
4613
4614 /*
4615 * See why we failed, and take corrective action.
4616 */
4617 switch (kr) {
4618 case VM_FAULT_MEMORY_SHORTAGE:
4619 if (vm_page_wait((change_wiring) ?
4620 THREAD_UNINT :
4621 THREAD_ABORTSAFE))
4622 goto RetryFault;
4623 /*
4624 * fall thru
4625 */
4626 case VM_FAULT_INTERRUPTED:
4627 kr = KERN_ABORTED;
4628 goto done;
4629 case VM_FAULT_RETRY:
4630 goto RetryFault;
4631 case VM_FAULT_MEMORY_ERROR:
4632 if (error_code)
4633 kr = error_code;
4634 else
4635 kr = KERN_MEMORY_ERROR;
4636 goto done;
4637 default:
4638 panic("vm_fault: unexpected error 0x%x from "
4639 "vm_fault_page()\n", kr);
4640 }
4641 }
4642 m = result_page;
4643 m_object = NULL;
4644
4645 if (m != VM_PAGE_NULL) {
4646 m_object = VM_PAGE_OBJECT(m);
4647 assert((change_wiring && !wired) ?
4648 (top_page == VM_PAGE_NULL) :
4649 ((top_page == VM_PAGE_NULL) == (m_object == object)));
4650 }
4651
4652 /*
4653 * What to do with the resulting page from vm_fault_page
4654 * if it doesn't get entered into the physical map:
4655 */
4656 #define RELEASE_PAGE(m) \
4657 MACRO_BEGIN \
4658 PAGE_WAKEUP_DONE(m); \
4659 if ( !VM_PAGE_PAGEABLE(m)) { \
4660 vm_page_lockspin_queues(); \
4661 if ( !VM_PAGE_PAGEABLE(m)) \
4662 vm_page_activate(m); \
4663 vm_page_unlock_queues(); \
4664 } \
4665 MACRO_END
4666
4667
4668 object_locks_dropped = FALSE;
4669 /*
4670 * We must verify that the maps have not changed
4671 * since our last lookup. vm_map_verify() needs the
4672 * map lock (shared) but we are holding object locks.
4673 * So we do a try_lock() first and, if that fails, we
4674 * drop the object locks and go in for the map lock again.
4675 */
4676 if (!vm_map_try_lock_read(original_map)) {
4677
4678 if (m != VM_PAGE_NULL) {
4679 old_copy_object = m_object->copy;
4680 vm_object_unlock(m_object);
4681 } else {
4682 old_copy_object = VM_OBJECT_NULL;
4683 vm_object_unlock(object);
4684 }
4685
4686 object_locks_dropped = TRUE;
4687
4688 vm_map_lock_read(original_map);
4689 }
4690
4691 if ((map != original_map) || !vm_map_verify(map, &version)) {
4692
4693 if (object_locks_dropped == FALSE) {
4694 if (m != VM_PAGE_NULL) {
4695 old_copy_object = m_object->copy;
4696 vm_object_unlock(m_object);
4697 } else {
4698 old_copy_object = VM_OBJECT_NULL;
4699 vm_object_unlock(object);
4700 }
4701
4702 object_locks_dropped = TRUE;
4703 }
4704
4705 /*
4706 * no object locks are held at this point
4707 */
4708 vm_object_t retry_object;
4709 vm_object_offset_t retry_offset;
4710 vm_prot_t retry_prot;
4711
4712 /*
4713 * To avoid trying to write_lock the map while another
4714 * thread has it read_locked (in vm_map_pageable), we
4715 * do not try for write permission. If the page is
4716 * still writable, we will get write permission. If it
4717 * is not, or has been marked needs_copy, we enter the
4718 * mapping without write permission, and will merely
4719 * take another fault.
4720 */
4721 map = original_map;
4722
4723 kr = vm_map_lookup_locked(&map, vaddr,
4724 fault_type & ~VM_PROT_WRITE,
4725 OBJECT_LOCK_EXCLUSIVE, &version,
4726 &retry_object, &retry_offset, &retry_prot,
4727 &wired,
4728 &fault_info,
4729 &real_map);
4730 pmap = real_map->pmap;
4731
4732 if (kr != KERN_SUCCESS) {
4733 vm_map_unlock_read(map);
4734
4735 if (m != VM_PAGE_NULL) {
4736 assert(VM_PAGE_OBJECT(m) == m_object);
4737
4738 /*
4739 * retake the lock so that
4740 * we can drop the paging reference
4741 * in vm_fault_cleanup and do the
4742 * PAGE_WAKEUP_DONE in RELEASE_PAGE
4743 */
4744 vm_object_lock(m_object);
4745
4746 RELEASE_PAGE(m);
4747
4748 vm_fault_cleanup(m_object, top_page);
4749 } else {
4750 /*
4751 * retake the lock so that
4752 * we can drop the paging reference
4753 * in vm_fault_cleanup
4754 */
4755 vm_object_lock(object);
4756
4757 vm_fault_cleanup(object, top_page);
4758 }
4759 vm_object_deallocate(object);
4760
4761 goto done;
4762 }
4763 vm_object_unlock(retry_object);
4764
4765 if ((retry_object != object) || (retry_offset != offset)) {
4766
4767 vm_map_unlock_read(map);
4768 if (real_map != map)
4769 vm_map_unlock(real_map);
4770
4771 if (m != VM_PAGE_NULL) {
4772 assert(VM_PAGE_OBJECT(m) == m_object);
4773
4774 /*
4775 * retake the lock so that
4776 * we can drop the paging reference
4777 * in vm_fault_cleanup and do the
4778 * PAGE_WAKEUP_DONE in RELEASE_PAGE
4779 */
4780 vm_object_lock(m_object);
4781
4782 RELEASE_PAGE(m);
4783
4784 vm_fault_cleanup(m_object, top_page);
4785 } else {
4786 /*
4787 * retake the lock so that
4788 * we can drop the paging reference
4789 * in vm_fault_cleanup
4790 */
4791 vm_object_lock(object);
4792
4793 vm_fault_cleanup(object, top_page);
4794 }
4795 vm_object_deallocate(object);
4796
4797 goto RetryFault;
4798 }
4799 /*
4800 * Check whether the protection has changed or the object
4801 * has been copied while we left the map unlocked.
4802 */
4803 if (pmap_has_prot_policy(retry_prot)) {
4804 /* If the pmap layer cares, pass the full set. */
4805 prot = retry_prot;
4806 } else {
4807 prot &= retry_prot;
4808 }
4809 }
4810
4811 if (object_locks_dropped == TRUE) {
4812 if (m != VM_PAGE_NULL) {
4813 vm_object_lock(m_object);
4814
4815 if (m_object->copy != old_copy_object) {
4816 /*
4817 * The copy object changed while the top-level object
4818 * was unlocked, so take away write permission.
4819 */
4820 assert(!pmap_has_prot_policy(prot));
4821 prot &= ~VM_PROT_WRITE;
4822 }
4823 } else
4824 vm_object_lock(object);
4825
4826 object_locks_dropped = FALSE;
4827 }
4828
4829 /*
4830 * If we want to wire down this page, but no longer have
4831 * adequate permissions, we must start all over.
4832 */
4833 if (wired && (fault_type != (prot | VM_PROT_WRITE))) {
4834
4835 vm_map_unlock_read(map);
4836 if (real_map != map)
4837 vm_map_unlock(real_map);
4838
4839 if (m != VM_PAGE_NULL) {
4840 assert(VM_PAGE_OBJECT(m) == m_object);
4841
4842 RELEASE_PAGE(m);
4843
4844 vm_fault_cleanup(m_object, top_page);
4845 } else
4846 vm_fault_cleanup(object, top_page);
4847
4848 vm_object_deallocate(object);
4849
4850 goto RetryFault;
4851 }
4852 if (m != VM_PAGE_NULL) {
4853 /*
4854 * Put this page into the physical map.
4855 * We had to do the unlock above because pmap_enter
4856 * may cause other faults. The page may be on
4857 * the pageout queues. If the pageout daemon comes
4858 * across the page, it will remove it from the queues.
4859 */
4860 if (caller_pmap) {
4861 kr = vm_fault_enter(m,
4862 caller_pmap,
4863 caller_pmap_addr,
4864 prot,
4865 caller_prot,
4866 wired,
4867 change_wiring,
4868 wire_tag,
4869 &fault_info,
4870 NULL,
4871 &type_of_fault);
4872 } else {
4873 kr = vm_fault_enter(m,
4874 pmap,
4875 vaddr,
4876 prot,
4877 caller_prot,
4878 wired,
4879 change_wiring,
4880 wire_tag,
4881 &fault_info,
4882 NULL,
4883 &type_of_fault);
4884 }
4885 assert(VM_PAGE_OBJECT(m) == m_object);
4886
4887 #if DEVELOPMENT || DEBUG
4888 {
4889 int event_code = 0;
4890
4891 if (m_object->internal)
4892 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_INTERNAL));
4893 else if (m_object->object_is_shared_cache)
4894 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_SHAREDCACHE));
4895 else
4896 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_EXTERNAL));
4897
4898 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, event_code, trace_real_vaddr, (fault_info.user_tag << 16) | (caller_prot << 8) | type_of_fault, m->vmp_offset, get_current_unique_pid(), 0);
4899
4900 DTRACE_VM6(real_fault, vm_map_offset_t, real_vaddr, vm_map_offset_t, m->vmp_offset, int, event_code, int, caller_prot, int, type_of_fault, int, fault_info.user_tag);
4901 }
4902 #endif
4903 if (kr != KERN_SUCCESS) {
4904 /* abort this page fault */
4905 vm_map_unlock_read(map);
4906 if (real_map != map)
4907 vm_map_unlock(real_map);
4908 PAGE_WAKEUP_DONE(m);
4909 vm_fault_cleanup(m_object, top_page);
4910 vm_object_deallocate(object);
4911 goto done;
4912 }
4913 if (physpage_p != NULL) {
4914 /* for vm_map_wire_and_extract() */
4915 *physpage_p = VM_PAGE_GET_PHYS_PAGE(m);
4916 if (prot & VM_PROT_WRITE) {
4917 vm_object_lock_assert_exclusive(m_object);
4918 m->vmp_dirty = TRUE;
4919 }
4920 }
4921 } else {
4922
4923 vm_map_entry_t entry;
4924 vm_map_offset_t laddr;
4925 vm_map_offset_t ldelta, hdelta;
4926
4927 /*
4928 * do a pmap block mapping from the physical address
4929 * in the object
4930 */
4931
4932 if (real_map != map)
4933 vm_map_unlock(real_map);
4934
4935 if (original_map != map) {
4936 vm_map_unlock_read(map);
4937 vm_map_lock_read(original_map);
4938 map = original_map;
4939 }
4940 real_map = map;
4941
4942 laddr = vaddr;
4943 hdelta = 0xFFFFF000;
4944 ldelta = 0xFFFFF000;
4945
4946 while (vm_map_lookup_entry(map, laddr, &entry)) {
4947 if (ldelta > (laddr - entry->vme_start))
4948 ldelta = laddr - entry->vme_start;
4949 if (hdelta > (entry->vme_end - laddr))
4950 hdelta = entry->vme_end - laddr;
4951 if (entry->is_sub_map) {
4952
4953 laddr = ((laddr - entry->vme_start)
4954 + VME_OFFSET(entry));
4955 vm_map_lock_read(VME_SUBMAP(entry));
4956
4957 if (map != real_map)
4958 vm_map_unlock_read(map);
4959 if (entry->use_pmap) {
4960 vm_map_unlock_read(real_map);
4961 real_map = VME_SUBMAP(entry);
4962 }
4963 map = VME_SUBMAP(entry);
4964
4965 } else {
4966 break;
4967 }
4968 }
4969
4970 if (vm_map_lookup_entry(map, laddr, &entry) &&
4971 (VME_OBJECT(entry) != NULL) &&
4972 (VME_OBJECT(entry) == object)) {
4973 int superpage;
4974
4975 if (!object->pager_created &&
4976 object->phys_contiguous &&
4977 VME_OFFSET(entry) == 0 &&
4978 (entry->vme_end - entry->vme_start == object->vo_size) &&
4979 VM_MAP_PAGE_ALIGNED(entry->vme_start, (object->vo_size-1))) {
4980 superpage = VM_MEM_SUPERPAGE;
4981 } else {
4982 superpage = 0;
4983 }
4984
4985 if (superpage && physpage_p) {
4986 /* for vm_map_wire_and_extract() */
4987 *physpage_p = (ppnum_t)
4988 ((((vm_map_offset_t)
4989 object->vo_shadow_offset)
4990 + VME_OFFSET(entry)
4991 + (laddr - entry->vme_start))
4992 >> PAGE_SHIFT);
4993 }
4994
4995 if (caller_pmap) {
4996 /*
4997 * Set up a block mapped area
4998 */
4999 assert((uint32_t)((ldelta + hdelta) >> PAGE_SHIFT) == ((ldelta + hdelta) >> PAGE_SHIFT));
5000 kr = pmap_map_block(caller_pmap,
5001 (addr64_t)(caller_pmap_addr - ldelta),
5002 (ppnum_t)((((vm_map_offset_t) (VME_OBJECT(entry)->vo_shadow_offset)) +
5003 VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT),
5004 (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot,
5005 (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0);
5006
5007 if (kr != KERN_SUCCESS) {
5008 goto cleanup;
5009 }
5010 } else {
5011 /*
5012 * Set up a block mapped area
5013 */
5014 assert((uint32_t)((ldelta + hdelta) >> PAGE_SHIFT) == ((ldelta + hdelta) >> PAGE_SHIFT));
5015 kr = pmap_map_block(real_map->pmap,
5016 (addr64_t)(vaddr - ldelta),
5017 (ppnum_t)((((vm_map_offset_t)(VME_OBJECT(entry)->vo_shadow_offset)) +
5018 VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT),
5019 (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot,
5020 (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0);
5021
5022 if (kr != KERN_SUCCESS) {
5023 goto cleanup;
5024 }
5025 }
5026 }
5027 }
5028
5029 /*
5030 * Success
5031 */
5032 kr = KERN_SUCCESS;
5033
5034 /*
5035 * TODO: could most of the done cases just use cleanup?
5036 */
5037 cleanup:
5038 /*
5039 * Unlock everything, and return
5040 */
5041 vm_map_unlock_read(map);
5042 if (real_map != map)
5043 vm_map_unlock(real_map);
5044
5045 if (m != VM_PAGE_NULL) {
5046 assert(VM_PAGE_OBJECT(m) == m_object);
5047
5048 if (!m_object->internal && (fault_type & VM_PROT_WRITE)) {
5049
5050 vm_object_paging_begin(m_object);
5051
5052 assert(written_on_object == VM_OBJECT_NULL);
5053 written_on_object = m_object;
5054 written_on_pager = m_object->pager;
5055 written_on_offset = m_object->paging_offset + m->vmp_offset;
5056 }
5057 PAGE_WAKEUP_DONE(m);
5058
5059 vm_fault_cleanup(m_object, top_page);
5060 } else
5061 vm_fault_cleanup(object, top_page);
5062
5063 vm_object_deallocate(object);
5064
5065 #undef RELEASE_PAGE
5066
5067 done:
5068 thread_interrupt_level(interruptible_state);
5069
5070 /*
5071 * Only I/O throttle on faults which cause a pagein/swapin.
5072 */
5073 if ((type_of_fault == DBG_PAGEIND_FAULT) || (type_of_fault == DBG_PAGEINV_FAULT) || (type_of_fault == DBG_COMPRESSOR_SWAPIN_FAULT)) {
5074 throttle_lowpri_io(1);
5075 } else {
5076 if (kr == KERN_SUCCESS && type_of_fault != DBG_CACHE_HIT_FAULT && type_of_fault != DBG_GUARD_FAULT) {
5077
5078 if ((throttle_delay = vm_page_throttled(TRUE))) {
5079
5080 if (vm_debug_events) {
5081 if (type_of_fault == DBG_COMPRESSOR_FAULT)
5082 VM_DEBUG_EVENT(vmf_compressordelay, VMF_COMPRESSORDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
5083 else if (type_of_fault == DBG_COW_FAULT)
5084 VM_DEBUG_EVENT(vmf_cowdelay, VMF_COWDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
5085 else
5086 VM_DEBUG_EVENT(vmf_zfdelay, VMF_ZFDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
5087 }
5088 delay(throttle_delay);
5089 }
5090 }
5091 }
5092
5093 if (written_on_object) {
5094
5095 vnode_pager_dirtied(written_on_pager, written_on_offset, written_on_offset + PAGE_SIZE_64);
5096
5097 vm_object_lock(written_on_object);
5098 vm_object_paging_end(written_on_object);
5099 vm_object_unlock(written_on_object);
5100
5101 written_on_object = VM_OBJECT_NULL;
5102 }
5103
5104 if (rtfault) {
5105 vm_record_rtfault(cthread, fstart, trace_vaddr, type_of_fault);
5106 }
5107
5108 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
5109 (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END,
5110 ((uint64_t)trace_vaddr >> 32),
5111 trace_vaddr,
5112 kr,
5113 type_of_fault,
5114 0);
5115
5116 return (kr);
5117 }
5118
5119 /*
5120 * vm_fault_wire:
5121 *
5122 * Wire down a range of virtual addresses in a map.
5123 */
5124 kern_return_t
5125 vm_fault_wire(
5126 vm_map_t map,
5127 vm_map_entry_t entry,
5128 vm_prot_t prot,
5129 vm_tag_t wire_tag,
5130 pmap_t pmap,
5131 vm_map_offset_t pmap_addr,
5132 ppnum_t *physpage_p)
5133 {
5134 vm_map_offset_t va;
5135 vm_map_offset_t end_addr = entry->vme_end;
5136 kern_return_t rc;
5137
5138 assert(entry->in_transition);
5139
5140 if ((VME_OBJECT(entry) != NULL) &&
5141 !entry->is_sub_map &&
5142 VME_OBJECT(entry)->phys_contiguous) {
5143 return KERN_SUCCESS;
5144 }
5145
5146 /*
5147 * Inform the physical mapping system that the
5148 * range of addresses may not fault, so that
5149 * page tables and such can be locked down as well.
5150 */
5151
5152 pmap_pageable(pmap, pmap_addr,
5153 pmap_addr + (end_addr - entry->vme_start), FALSE);
5154
5155 /*
5156 * We simulate a fault to get the page and enter it
5157 * in the physical map.
5158 */
5159
5160 for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
5161 rc = vm_fault_wire_fast(map, va, prot, wire_tag, entry, pmap,
5162 pmap_addr + (va - entry->vme_start),
5163 physpage_p);
5164 if (rc != KERN_SUCCESS) {
5165 rc = vm_fault_internal(map, va, prot, TRUE, wire_tag,
5166 ((pmap == kernel_pmap)
5167 ? THREAD_UNINT
5168 : THREAD_ABORTSAFE),
5169 pmap,
5170 (pmap_addr +
5171 (va - entry->vme_start)),
5172 physpage_p);
5173 DTRACE_VM2(softlock, int, 1, (uint64_t *), NULL);
5174 }
5175
5176 if (rc != KERN_SUCCESS) {
5177 struct vm_map_entry tmp_entry = *entry;
5178
5179 /* unwire wired pages */
5180 tmp_entry.vme_end = va;
5181 vm_fault_unwire(map,
5182 &tmp_entry, FALSE, pmap, pmap_addr);
5183
5184 return rc;
5185 }
5186 }
5187 return KERN_SUCCESS;
5188 }
5189
5190 /*
5191 * vm_fault_unwire:
5192 *
5193 * Unwire a range of virtual addresses in a map.
5194 */
5195 void
5196 vm_fault_unwire(
5197 vm_map_t map,
5198 vm_map_entry_t entry,
5199 boolean_t deallocate,
5200 pmap_t pmap,
5201 vm_map_offset_t pmap_addr)
5202 {
5203 vm_map_offset_t va;
5204 vm_map_offset_t end_addr = entry->vme_end;
5205 vm_object_t object;
5206 struct vm_object_fault_info fault_info = {};
5207 unsigned int unwired_pages;
5208
5209 object = (entry->is_sub_map) ? VM_OBJECT_NULL : VME_OBJECT(entry);
5210
5211 /*
5212 * If it's marked phys_contiguous, then vm_fault_wire() didn't actually
5213 * do anything since such memory is wired by default. So we don't have
5214 * anything to undo here.
5215 */
5216
5217 if (object != VM_OBJECT_NULL && object->phys_contiguous)
5218 return;
5219
5220 fault_info.interruptible = THREAD_UNINT;
5221 fault_info.behavior = entry->behavior;
5222 fault_info.user_tag = VME_ALIAS(entry);
5223 if (entry->iokit_acct ||
5224 (!entry->is_sub_map && !entry->use_pmap)) {
5225 fault_info.pmap_options |= PMAP_OPTIONS_ALT_ACCT;
5226 }
5227 fault_info.lo_offset = VME_OFFSET(entry);
5228 fault_info.hi_offset = (entry->vme_end - entry->vme_start) + VME_OFFSET(entry);
5229 fault_info.no_cache = entry->no_cache;
5230 fault_info.stealth = TRUE;
5231
5232 unwired_pages = 0;
5233
5234 /*
5235 * Since the pages are wired down, we must be able to
5236 * get their mappings from the physical map system.
5237 */
5238
5239 for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
5240
5241 if (object == VM_OBJECT_NULL) {
5242 if (pmap) {
5243 pmap_change_wiring(pmap,
5244 pmap_addr + (va - entry->vme_start), FALSE);
5245 }
5246 (void) vm_fault(map, va, VM_PROT_NONE,
5247 TRUE, VM_KERN_MEMORY_NONE, THREAD_UNINT, pmap, pmap_addr);
5248 } else {
5249 vm_prot_t prot;
5250 vm_page_t result_page;
5251 vm_page_t top_page;
5252 vm_object_t result_object;
5253 vm_fault_return_t result;
5254
5255 /* cap cluster size at maximum UPL size */
5256 upl_size_t cluster_size;
5257 if (os_sub_overflow(end_addr, va, &cluster_size)) {
5258 cluster_size = 0 - (upl_size_t)PAGE_SIZE;
5259 }
5260 fault_info.cluster_size = cluster_size;
5261
5262 do {
5263 prot = VM_PROT_NONE;
5264
5265 vm_object_lock(object);
5266 vm_object_paging_begin(object);
5267 XPR(XPR_VM_FAULT,
5268 "vm_fault_unwire -> vm_fault_page\n",
5269 0,0,0,0,0);
5270 result_page = VM_PAGE_NULL;
5271 result = vm_fault_page(
5272 object,
5273 (VME_OFFSET(entry) +
5274 (va - entry->vme_start)),
5275 VM_PROT_NONE, TRUE,
5276 FALSE, /* page not looked up */
5277 &prot, &result_page, &top_page,
5278 (int *)0,
5279 NULL, map->no_zero_fill,
5280 FALSE, &fault_info);
5281 } while (result == VM_FAULT_RETRY);
5282
5283 /*
5284 * If this was a mapping to a file on a device that has been forcibly
5285 * unmounted, then we won't get a page back from vm_fault_page(). Just
5286 * move on to the next one in case the remaining pages are mapped from
5287 * different objects. During a forced unmount, the object is terminated
5288 * so the alive flag will be false if this happens. A forced unmount will
5289 * will occur when an external disk is unplugged before the user does an
5290 * eject, so we don't want to panic in that situation.
5291 */
5292
5293 if (result == VM_FAULT_MEMORY_ERROR && !object->alive)
5294 continue;
5295
5296 if (result == VM_FAULT_MEMORY_ERROR &&
5297 object == kernel_object) {
5298 /*
5299 * This must have been allocated with
5300 * KMA_KOBJECT and KMA_VAONLY and there's
5301 * no physical page at this offset.
5302 * We're done (no page to free).
5303 */
5304 assert(deallocate);
5305 continue;
5306 }
5307
5308 if (result != VM_FAULT_SUCCESS)
5309 panic("vm_fault_unwire: failure");
5310
5311 result_object = VM_PAGE_OBJECT(result_page);
5312
5313 if (deallocate) {
5314 assert(VM_PAGE_GET_PHYS_PAGE(result_page) !=
5315 vm_page_fictitious_addr);
5316 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(result_page));
5317 if (VM_PAGE_WIRED(result_page)) {
5318 unwired_pages++;
5319 }
5320 VM_PAGE_FREE(result_page);
5321 } else {
5322 if ((pmap) && (VM_PAGE_GET_PHYS_PAGE(result_page) != vm_page_guard_addr))
5323 pmap_change_wiring(pmap,
5324 pmap_addr + (va - entry->vme_start), FALSE);
5325
5326
5327 if (VM_PAGE_WIRED(result_page)) {
5328 vm_page_lockspin_queues();
5329 vm_page_unwire(result_page, TRUE);
5330 vm_page_unlock_queues();
5331 unwired_pages++;
5332 }
5333 if(entry->zero_wired_pages) {
5334 pmap_zero_page(VM_PAGE_GET_PHYS_PAGE(result_page));
5335 entry->zero_wired_pages = FALSE;
5336 }
5337
5338 PAGE_WAKEUP_DONE(result_page);
5339 }
5340 vm_fault_cleanup(result_object, top_page);
5341 }
5342 }
5343
5344 /*
5345 * Inform the physical mapping system that the range
5346 * of addresses may fault, so that page tables and
5347 * such may be unwired themselves.
5348 */
5349
5350 pmap_pageable(pmap, pmap_addr,
5351 pmap_addr + (end_addr - entry->vme_start), TRUE);
5352
5353 if (kernel_object == object) {
5354 vm_tag_update_size(fault_info.user_tag, -ptoa_64(unwired_pages));
5355 }
5356 }
5357
5358 /*
5359 * vm_fault_wire_fast:
5360 *
5361 * Handle common case of a wire down page fault at the given address.
5362 * If successful, the page is inserted into the associated physical map.
5363 * The map entry is passed in to avoid the overhead of a map lookup.
5364 *
5365 * NOTE: the given address should be truncated to the
5366 * proper page address.
5367 *
5368 * KERN_SUCCESS is returned if the page fault is handled; otherwise,
5369 * a standard error specifying why the fault is fatal is returned.
5370 *
5371 * The map in question must be referenced, and remains so.
5372 * Caller has a read lock on the map.
5373 *
5374 * This is a stripped version of vm_fault() for wiring pages. Anything
5375 * other than the common case will return KERN_FAILURE, and the caller
5376 * is expected to call vm_fault().
5377 */
5378 static kern_return_t
5379 vm_fault_wire_fast(
5380 __unused vm_map_t map,
5381 vm_map_offset_t va,
5382 __unused vm_prot_t caller_prot,
5383 vm_tag_t wire_tag,
5384 vm_map_entry_t entry,
5385 pmap_t pmap,
5386 vm_map_offset_t pmap_addr,
5387 ppnum_t *physpage_p)
5388 {
5389 vm_object_t object;
5390 vm_object_offset_t offset;
5391 vm_page_t m;
5392 vm_prot_t prot;
5393 thread_t thread = current_thread();
5394 int type_of_fault;
5395 kern_return_t kr;
5396 struct vm_object_fault_info fault_info = {};
5397
5398 VM_STAT_INCR(faults);
5399
5400 if (thread != THREAD_NULL && thread->task != TASK_NULL)
5401 thread->task->faults++;
5402
5403 /*
5404 * Recovery actions
5405 */
5406
5407 #undef RELEASE_PAGE
5408 #define RELEASE_PAGE(m) { \
5409 PAGE_WAKEUP_DONE(m); \
5410 vm_page_lockspin_queues(); \
5411 vm_page_unwire(m, TRUE); \
5412 vm_page_unlock_queues(); \
5413 }
5414
5415
5416 #undef UNLOCK_THINGS
5417 #define UNLOCK_THINGS { \
5418 vm_object_paging_end(object); \
5419 vm_object_unlock(object); \
5420 }
5421
5422 #undef UNLOCK_AND_DEALLOCATE
5423 #define UNLOCK_AND_DEALLOCATE { \
5424 UNLOCK_THINGS; \
5425 vm_object_deallocate(object); \
5426 }
5427 /*
5428 * Give up and have caller do things the hard way.
5429 */
5430
5431 #define GIVE_UP { \
5432 UNLOCK_AND_DEALLOCATE; \
5433 return(KERN_FAILURE); \
5434 }
5435
5436
5437 /*
5438 * If this entry is not directly to a vm_object, bail out.
5439 */
5440 if (entry->is_sub_map) {
5441 assert(physpage_p == NULL);
5442 return(KERN_FAILURE);
5443 }
5444
5445 /*
5446 * Find the backing store object and offset into it.
5447 */
5448
5449 object = VME_OBJECT(entry);
5450 offset = (va - entry->vme_start) + VME_OFFSET(entry);
5451 prot = entry->protection;
5452
5453 /*
5454 * Make a reference to this object to prevent its
5455 * disposal while we are messing with it.
5456 */
5457
5458 vm_object_lock(object);
5459 vm_object_reference_locked(object);
5460 vm_object_paging_begin(object);
5461
5462 /*
5463 * INVARIANTS (through entire routine):
5464 *
5465 * 1) At all times, we must either have the object
5466 * lock or a busy page in some object to prevent
5467 * some other thread from trying to bring in
5468 * the same page.
5469 *
5470 * 2) Once we have a busy page, we must remove it from
5471 * the pageout queues, so that the pageout daemon
5472 * will not grab it away.
5473 *
5474 */
5475
5476 /*
5477 * Look for page in top-level object. If it's not there or
5478 * there's something going on, give up.
5479 */
5480 m = vm_page_lookup(object, offset);
5481 if ((m == VM_PAGE_NULL) || (m->vmp_busy) ||
5482 (m->vmp_unusual && ( m->vmp_error || m->vmp_restart || m->vmp_absent))) {
5483
5484 GIVE_UP;
5485 }
5486 if (m->vmp_fictitious &&
5487 VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
5488 /*
5489 * Guard pages are fictitious pages and are never
5490 * entered into a pmap, so let's say it's been wired...
5491 */
5492 kr = KERN_SUCCESS;
5493 goto done;
5494 }
5495
5496 /*
5497 * Wire the page down now. All bail outs beyond this
5498 * point must unwire the page.
5499 */
5500
5501 vm_page_lockspin_queues();
5502 vm_page_wire(m, wire_tag, TRUE);
5503 vm_page_unlock_queues();
5504
5505 /*
5506 * Mark page busy for other threads.
5507 */
5508 assert(!m->vmp_busy);
5509 m->vmp_busy = TRUE;
5510 assert(!m->vmp_absent);
5511
5512 /*
5513 * Give up if the page is being written and there's a copy object
5514 */
5515 if ((object->copy != VM_OBJECT_NULL) && (prot & VM_PROT_WRITE)) {
5516 RELEASE_PAGE(m);
5517 GIVE_UP;
5518 }
5519
5520 fault_info.user_tag = VME_ALIAS(entry);
5521 fault_info.pmap_options = 0;
5522 if (entry->iokit_acct ||
5523 (!entry->is_sub_map && !entry->use_pmap)) {
5524 fault_info.pmap_options |= PMAP_OPTIONS_ALT_ACCT;
5525 }
5526
5527 /*
5528 * Put this page into the physical map.
5529 */
5530 type_of_fault = DBG_CACHE_HIT_FAULT;
5531 kr = vm_fault_enter(m,
5532 pmap,
5533 pmap_addr,
5534 prot,
5535 prot,
5536 TRUE, /* wired */
5537 FALSE, /* change_wiring */
5538 wire_tag,
5539 &fault_info,
5540 NULL,
5541 &type_of_fault);
5542 if (kr != KERN_SUCCESS) {
5543 RELEASE_PAGE(m);
5544 GIVE_UP;
5545 }
5546
5547 done:
5548 /*
5549 * Unlock everything, and return
5550 */
5551
5552 if (physpage_p) {
5553 /* for vm_map_wire_and_extract() */
5554 if (kr == KERN_SUCCESS) {
5555 assert(object == VM_PAGE_OBJECT(m));
5556 *physpage_p = VM_PAGE_GET_PHYS_PAGE(m);
5557 if (prot & VM_PROT_WRITE) {
5558 vm_object_lock_assert_exclusive(object);
5559 m->vmp_dirty = TRUE;
5560 }
5561 } else {
5562 *physpage_p = 0;
5563 }
5564 }
5565
5566 PAGE_WAKEUP_DONE(m);
5567 UNLOCK_AND_DEALLOCATE;
5568
5569 return kr;
5570
5571 }
5572
5573 /*
5574 * Routine: vm_fault_copy_cleanup
5575 * Purpose:
5576 * Release a page used by vm_fault_copy.
5577 */
5578
5579 static void
5580 vm_fault_copy_cleanup(
5581 vm_page_t page,
5582 vm_page_t top_page)
5583 {
5584 vm_object_t object = VM_PAGE_OBJECT(page);
5585
5586 vm_object_lock(object);
5587 PAGE_WAKEUP_DONE(page);
5588 if ( !VM_PAGE_PAGEABLE(page)) {
5589 vm_page_lockspin_queues();
5590 if ( !VM_PAGE_PAGEABLE(page)) {
5591 vm_page_activate(page);
5592 }
5593 vm_page_unlock_queues();
5594 }
5595 vm_fault_cleanup(object, top_page);
5596 }
5597
5598 static void
5599 vm_fault_copy_dst_cleanup(
5600 vm_page_t page)
5601 {
5602 vm_object_t object;
5603
5604 if (page != VM_PAGE_NULL) {
5605 object = VM_PAGE_OBJECT(page);
5606 vm_object_lock(object);
5607 vm_page_lockspin_queues();
5608 vm_page_unwire(page, TRUE);
5609 vm_page_unlock_queues();
5610 vm_object_paging_end(object);
5611 vm_object_unlock(object);
5612 }
5613 }
5614
5615 /*
5616 * Routine: vm_fault_copy
5617 *
5618 * Purpose:
5619 * Copy pages from one virtual memory object to another --
5620 * neither the source nor destination pages need be resident.
5621 *
5622 * Before actually copying a page, the version associated with
5623 * the destination address map wil be verified.
5624 *
5625 * In/out conditions:
5626 * The caller must hold a reference, but not a lock, to
5627 * each of the source and destination objects and to the
5628 * destination map.
5629 *
5630 * Results:
5631 * Returns KERN_SUCCESS if no errors were encountered in
5632 * reading or writing the data. Returns KERN_INTERRUPTED if
5633 * the operation was interrupted (only possible if the
5634 * "interruptible" argument is asserted). Other return values
5635 * indicate a permanent error in copying the data.
5636 *
5637 * The actual amount of data copied will be returned in the
5638 * "copy_size" argument. In the event that the destination map
5639 * verification failed, this amount may be less than the amount
5640 * requested.
5641 */
5642 kern_return_t
5643 vm_fault_copy(
5644 vm_object_t src_object,
5645 vm_object_offset_t src_offset,
5646 vm_map_size_t *copy_size, /* INOUT */
5647 vm_object_t dst_object,
5648 vm_object_offset_t dst_offset,
5649 vm_map_t dst_map,
5650 vm_map_version_t *dst_version,
5651 int interruptible)
5652 {
5653 vm_page_t result_page;
5654
5655 vm_page_t src_page;
5656 vm_page_t src_top_page;
5657 vm_prot_t src_prot;
5658
5659 vm_page_t dst_page;
5660 vm_page_t dst_top_page;
5661 vm_prot_t dst_prot;
5662
5663 vm_map_size_t amount_left;
5664 vm_object_t old_copy_object;
5665 vm_object_t result_page_object = NULL;
5666 kern_return_t error = 0;
5667 vm_fault_return_t result;
5668
5669 vm_map_size_t part_size;
5670 struct vm_object_fault_info fault_info_src = {};
5671 struct vm_object_fault_info fault_info_dst = {};
5672
5673 /*
5674 * In order not to confuse the clustered pageins, align
5675 * the different offsets on a page boundary.
5676 */
5677
5678 #define RETURN(x) \
5679 MACRO_BEGIN \
5680 *copy_size -= amount_left; \
5681 MACRO_RETURN(x); \
5682 MACRO_END
5683
5684 amount_left = *copy_size;
5685
5686 fault_info_src.interruptible = interruptible;
5687 fault_info_src.behavior = VM_BEHAVIOR_SEQUENTIAL;
5688 fault_info_src.lo_offset = vm_object_trunc_page(src_offset);
5689 fault_info_src.hi_offset = fault_info_src.lo_offset + amount_left;
5690 fault_info_src.stealth = TRUE;
5691
5692 fault_info_dst.interruptible = interruptible;
5693 fault_info_dst.behavior = VM_BEHAVIOR_SEQUENTIAL;
5694 fault_info_dst.lo_offset = vm_object_trunc_page(dst_offset);
5695 fault_info_dst.hi_offset = fault_info_dst.lo_offset + amount_left;
5696 fault_info_dst.stealth = TRUE;
5697
5698 do { /* while (amount_left > 0) */
5699 /*
5700 * There may be a deadlock if both source and destination
5701 * pages are the same. To avoid this deadlock, the copy must
5702 * start by getting the destination page in order to apply
5703 * COW semantics if any.
5704 */
5705
5706 RetryDestinationFault: ;
5707
5708 dst_prot = VM_PROT_WRITE|VM_PROT_READ;
5709
5710 vm_object_lock(dst_object);
5711 vm_object_paging_begin(dst_object);
5712
5713 /* cap cluster size at maximum UPL size */
5714 upl_size_t cluster_size;
5715 if (os_convert_overflow(amount_left, &cluster_size)) {
5716 cluster_size = 0 - (upl_size_t)PAGE_SIZE;
5717 }
5718 fault_info_dst.cluster_size = cluster_size;
5719
5720 XPR(XPR_VM_FAULT,"vm_fault_copy -> vm_fault_page\n",0,0,0,0,0);
5721 dst_page = VM_PAGE_NULL;
5722 result = vm_fault_page(dst_object,
5723 vm_object_trunc_page(dst_offset),
5724 VM_PROT_WRITE|VM_PROT_READ,
5725 FALSE,
5726 FALSE, /* page not looked up */
5727 &dst_prot, &dst_page, &dst_top_page,
5728 (int *)0,
5729 &error,
5730 dst_map->no_zero_fill,
5731 FALSE, &fault_info_dst);
5732 switch (result) {
5733 case VM_FAULT_SUCCESS:
5734 break;
5735 case VM_FAULT_RETRY:
5736 goto RetryDestinationFault;
5737 case VM_FAULT_MEMORY_SHORTAGE:
5738 if (vm_page_wait(interruptible))
5739 goto RetryDestinationFault;
5740 /* fall thru */
5741 case VM_FAULT_INTERRUPTED:
5742 RETURN(MACH_SEND_INTERRUPTED);
5743 case VM_FAULT_SUCCESS_NO_VM_PAGE:
5744 /* success but no VM page: fail the copy */
5745 vm_object_paging_end(dst_object);
5746 vm_object_unlock(dst_object);
5747 /*FALLTHROUGH*/
5748 case VM_FAULT_MEMORY_ERROR:
5749 if (error)
5750 return (error);
5751 else
5752 return(KERN_MEMORY_ERROR);
5753 default:
5754 panic("vm_fault_copy: unexpected error 0x%x from "
5755 "vm_fault_page()\n", result);
5756 }
5757 assert ((dst_prot & VM_PROT_WRITE) != VM_PROT_NONE);
5758
5759 assert(dst_object == VM_PAGE_OBJECT(dst_page));
5760 old_copy_object = dst_object->copy;
5761
5762 /*
5763 * There exists the possiblity that the source and
5764 * destination page are the same. But we can't
5765 * easily determine that now. If they are the
5766 * same, the call to vm_fault_page() for the
5767 * destination page will deadlock. To prevent this we
5768 * wire the page so we can drop busy without having
5769 * the page daemon steal the page. We clean up the
5770 * top page but keep the paging reference on the object
5771 * holding the dest page so it doesn't go away.
5772 */
5773
5774 vm_page_lockspin_queues();
5775 vm_page_wire(dst_page, VM_KERN_MEMORY_OSFMK, TRUE);
5776 vm_page_unlock_queues();
5777 PAGE_WAKEUP_DONE(dst_page);
5778 vm_object_unlock(dst_object);
5779
5780 if (dst_top_page != VM_PAGE_NULL) {
5781 vm_object_lock(dst_object);
5782 VM_PAGE_FREE(dst_top_page);
5783 vm_object_paging_end(dst_object);
5784 vm_object_unlock(dst_object);
5785 }
5786
5787 RetrySourceFault: ;
5788
5789 if (src_object == VM_OBJECT_NULL) {
5790 /*
5791 * No source object. We will just
5792 * zero-fill the page in dst_object.
5793 */
5794 src_page = VM_PAGE_NULL;
5795 result_page = VM_PAGE_NULL;
5796 } else {
5797 vm_object_lock(src_object);
5798 src_page = vm_page_lookup(src_object,
5799 vm_object_trunc_page(src_offset));
5800 if (src_page == dst_page) {
5801 src_prot = dst_prot;
5802 result_page = VM_PAGE_NULL;
5803 } else {
5804 src_prot = VM_PROT_READ;
5805 vm_object_paging_begin(src_object);
5806
5807 /* cap cluster size at maximum UPL size */
5808 if (os_convert_overflow(amount_left, &cluster_size)) {
5809 cluster_size = 0 - (upl_size_t)PAGE_SIZE;
5810 }
5811 fault_info_src.cluster_size = cluster_size;
5812
5813 XPR(XPR_VM_FAULT,
5814 "vm_fault_copy(2) -> vm_fault_page\n",
5815 0,0,0,0,0);
5816 result_page = VM_PAGE_NULL;
5817 result = vm_fault_page(
5818 src_object,
5819 vm_object_trunc_page(src_offset),
5820 VM_PROT_READ, FALSE,
5821 FALSE, /* page not looked up */
5822 &src_prot,
5823 &result_page, &src_top_page,
5824 (int *)0, &error, FALSE,
5825 FALSE, &fault_info_src);
5826
5827 switch (result) {
5828 case VM_FAULT_SUCCESS:
5829 break;
5830 case VM_FAULT_RETRY:
5831 goto RetrySourceFault;
5832 case VM_FAULT_MEMORY_SHORTAGE:
5833 if (vm_page_wait(interruptible))
5834 goto RetrySourceFault;
5835 /* fall thru */
5836 case VM_FAULT_INTERRUPTED:
5837 vm_fault_copy_dst_cleanup(dst_page);
5838 RETURN(MACH_SEND_INTERRUPTED);
5839 case VM_FAULT_SUCCESS_NO_VM_PAGE:
5840 /* success but no VM page: fail */
5841 vm_object_paging_end(src_object);
5842 vm_object_unlock(src_object);
5843 /*FALLTHROUGH*/
5844 case VM_FAULT_MEMORY_ERROR:
5845 vm_fault_copy_dst_cleanup(dst_page);
5846 if (error)
5847 return (error);
5848 else
5849 return(KERN_MEMORY_ERROR);
5850 default:
5851 panic("vm_fault_copy(2): unexpected "
5852 "error 0x%x from "
5853 "vm_fault_page()\n", result);
5854 }
5855
5856 result_page_object = VM_PAGE_OBJECT(result_page);
5857 assert((src_top_page == VM_PAGE_NULL) ==
5858 (result_page_object == src_object));
5859 }
5860 assert ((src_prot & VM_PROT_READ) != VM_PROT_NONE);
5861 vm_object_unlock(result_page_object);
5862 }
5863
5864 vm_map_lock_read(dst_map);
5865
5866 if (!vm_map_verify(dst_map, dst_version)) {
5867 vm_map_unlock_read(dst_map);
5868 if (result_page != VM_PAGE_NULL && src_page != dst_page)
5869 vm_fault_copy_cleanup(result_page, src_top_page);
5870 vm_fault_copy_dst_cleanup(dst_page);
5871 break;
5872 }
5873 assert(dst_object == VM_PAGE_OBJECT(dst_page));
5874
5875 vm_object_lock(dst_object);
5876
5877 if (dst_object->copy != old_copy_object) {
5878 vm_object_unlock(dst_object);
5879 vm_map_unlock_read(dst_map);
5880 if (result_page != VM_PAGE_NULL && src_page != dst_page)
5881 vm_fault_copy_cleanup(result_page, src_top_page);
5882 vm_fault_copy_dst_cleanup(dst_page);
5883 break;
5884 }
5885 vm_object_unlock(dst_object);
5886
5887 /*
5888 * Copy the page, and note that it is dirty
5889 * immediately.
5890 */
5891
5892 if (!page_aligned(src_offset) ||
5893 !page_aligned(dst_offset) ||
5894 !page_aligned(amount_left)) {
5895
5896 vm_object_offset_t src_po,
5897 dst_po;
5898
5899 src_po = src_offset - vm_object_trunc_page(src_offset);
5900 dst_po = dst_offset - vm_object_trunc_page(dst_offset);
5901
5902 if (dst_po > src_po) {
5903 part_size = PAGE_SIZE - dst_po;
5904 } else {
5905 part_size = PAGE_SIZE - src_po;
5906 }
5907 if (part_size > (amount_left)){
5908 part_size = amount_left;
5909 }
5910
5911 if (result_page == VM_PAGE_NULL) {
5912 assert((vm_offset_t) dst_po == dst_po);
5913 assert((vm_size_t) part_size == part_size);
5914 vm_page_part_zero_fill(dst_page,
5915 (vm_offset_t) dst_po,
5916 (vm_size_t) part_size);
5917 } else {
5918 assert((vm_offset_t) src_po == src_po);
5919 assert((vm_offset_t) dst_po == dst_po);
5920 assert((vm_size_t) part_size == part_size);
5921 vm_page_part_copy(result_page,
5922 (vm_offset_t) src_po,
5923 dst_page,
5924 (vm_offset_t) dst_po,
5925 (vm_size_t)part_size);
5926 if(!dst_page->vmp_dirty){
5927 vm_object_lock(dst_object);
5928 SET_PAGE_DIRTY(dst_page, TRUE);
5929 vm_object_unlock(dst_object);
5930 }
5931
5932 }
5933 } else {
5934 part_size = PAGE_SIZE;
5935
5936 if (result_page == VM_PAGE_NULL)
5937 vm_page_zero_fill(dst_page);
5938 else{
5939 vm_object_lock(result_page_object);
5940 vm_page_copy(result_page, dst_page);
5941 vm_object_unlock(result_page_object);
5942
5943 if(!dst_page->vmp_dirty){
5944 vm_object_lock(dst_object);
5945 SET_PAGE_DIRTY(dst_page, TRUE);
5946 vm_object_unlock(dst_object);
5947 }
5948 }
5949
5950 }
5951
5952 /*
5953 * Unlock everything, and return
5954 */
5955
5956 vm_map_unlock_read(dst_map);
5957
5958 if (result_page != VM_PAGE_NULL && src_page != dst_page)
5959 vm_fault_copy_cleanup(result_page, src_top_page);
5960 vm_fault_copy_dst_cleanup(dst_page);
5961
5962 amount_left -= part_size;
5963 src_offset += part_size;
5964 dst_offset += part_size;
5965 } while (amount_left > 0);
5966
5967 RETURN(KERN_SUCCESS);
5968 #undef RETURN
5969
5970 /*NOTREACHED*/
5971 }
5972
5973 #if VM_FAULT_CLASSIFY
5974 /*
5975 * Temporary statistics gathering support.
5976 */
5977
5978 /*
5979 * Statistics arrays:
5980 */
5981 #define VM_FAULT_TYPES_MAX 5
5982 #define VM_FAULT_LEVEL_MAX 8
5983
5984 int vm_fault_stats[VM_FAULT_TYPES_MAX][VM_FAULT_LEVEL_MAX];
5985
5986 #define VM_FAULT_TYPE_ZERO_FILL 0
5987 #define VM_FAULT_TYPE_MAP_IN 1
5988 #define VM_FAULT_TYPE_PAGER 2
5989 #define VM_FAULT_TYPE_COPY 3
5990 #define VM_FAULT_TYPE_OTHER 4
5991
5992
5993 void
5994 vm_fault_classify(vm_object_t object,
5995 vm_object_offset_t offset,
5996 vm_prot_t fault_type)
5997 {
5998 int type, level = 0;
5999 vm_page_t m;
6000
6001 while (TRUE) {
6002 m = vm_page_lookup(object, offset);
6003 if (m != VM_PAGE_NULL) {
6004 if (m->vmp_busy || m->vmp_error || m->vmp_restart || m->vmp_absent) {
6005 type = VM_FAULT_TYPE_OTHER;
6006 break;
6007 }
6008 if (((fault_type & VM_PROT_WRITE) == 0) ||
6009 ((level == 0) && object->copy == VM_OBJECT_NULL)) {
6010 type = VM_FAULT_TYPE_MAP_IN;
6011 break;
6012 }
6013 type = VM_FAULT_TYPE_COPY;
6014 break;
6015 }
6016 else {
6017 if (object->pager_created) {
6018 type = VM_FAULT_TYPE_PAGER;
6019 break;
6020 }
6021 if (object->shadow == VM_OBJECT_NULL) {
6022 type = VM_FAULT_TYPE_ZERO_FILL;
6023 break;
6024 }
6025
6026 offset += object->vo_shadow_offset;
6027 object = object->shadow;
6028 level++;
6029 continue;
6030 }
6031 }
6032
6033 if (level > VM_FAULT_LEVEL_MAX)
6034 level = VM_FAULT_LEVEL_MAX;
6035
6036 vm_fault_stats[type][level] += 1;
6037
6038 return;
6039 }
6040
6041 /* cleanup routine to call from debugger */
6042
6043 void
6044 vm_fault_classify_init(void)
6045 {
6046 int type, level;
6047
6048 for (type = 0; type < VM_FAULT_TYPES_MAX; type++) {
6049 for (level = 0; level < VM_FAULT_LEVEL_MAX; level++) {
6050 vm_fault_stats[type][level] = 0;
6051 }
6052 }
6053
6054 return;
6055 }
6056 #endif /* VM_FAULT_CLASSIFY */
6057
6058 vm_offset_t
6059 kdp_lightweight_fault(vm_map_t map, vm_offset_t cur_target_addr)
6060 {
6061 vm_map_entry_t entry;
6062 vm_object_t object;
6063 vm_offset_t object_offset;
6064 vm_page_t m;
6065 int compressor_external_state, compressed_count_delta;
6066 int compressor_flags = (C_DONT_BLOCK | C_KEEP | C_KDP);
6067 int my_fault_type = VM_PROT_READ;
6068 kern_return_t kr;
6069
6070 if (not_in_kdp) {
6071 panic("kdp_lightweight_fault called from outside of debugger context");
6072 }
6073
6074 assert(map != VM_MAP_NULL);
6075
6076 assert((cur_target_addr & PAGE_MASK) == 0);
6077 if ((cur_target_addr & PAGE_MASK) != 0) {
6078 return 0;
6079 }
6080
6081 if (kdp_lck_rw_lock_is_acquired_exclusive(&map->lock)) {
6082 return 0;
6083 }
6084
6085 if (!vm_map_lookup_entry(map, cur_target_addr, &entry)) {
6086 return 0;
6087 }
6088
6089 if (entry->is_sub_map) {
6090 return 0;
6091 }
6092
6093 object = VME_OBJECT(entry);
6094 if (object == VM_OBJECT_NULL) {
6095 return 0;
6096 }
6097
6098 object_offset = cur_target_addr - entry->vme_start + VME_OFFSET(entry);
6099
6100 while (TRUE) {
6101 if (kdp_lck_rw_lock_is_acquired_exclusive(&object->Lock)) {
6102 return 0;
6103 }
6104
6105 if (object->pager_created && (object->paging_in_progress ||
6106 object->activity_in_progress)) {
6107 return 0;
6108 }
6109
6110 m = kdp_vm_page_lookup(object, object_offset);
6111
6112 if (m != VM_PAGE_NULL) {
6113
6114 if ((object->wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
6115 return 0;
6116 }
6117
6118 if (m->vmp_laundry || m->vmp_busy || m->vmp_free_when_done || m->vmp_absent || m->vmp_error || m->vmp_cleaning ||
6119 m->vmp_overwriting || m->vmp_restart || m->vmp_unusual) {
6120 return 0;
6121 }
6122
6123 assert(!m->vmp_private);
6124 if (m->vmp_private) {
6125 return 0;
6126 }
6127
6128 assert(!m->vmp_fictitious);
6129 if (m->vmp_fictitious) {
6130 return 0;
6131 }
6132
6133 assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
6134 if (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
6135 return 0;
6136 }
6137
6138 return ptoa(VM_PAGE_GET_PHYS_PAGE(m));
6139 }
6140
6141 compressor_external_state = VM_EXTERNAL_STATE_UNKNOWN;
6142
6143 if (object->pager_created && MUST_ASK_PAGER(object, object_offset, compressor_external_state)) {
6144 if (compressor_external_state == VM_EXTERNAL_STATE_EXISTS) {
6145 kr = vm_compressor_pager_get(object->pager, (object_offset + object->paging_offset),
6146 kdp_compressor_decompressed_page_ppnum, &my_fault_type,
6147 compressor_flags, &compressed_count_delta);
6148 if (kr == KERN_SUCCESS) {
6149 return kdp_compressor_decompressed_page_paddr;
6150 } else {
6151 return 0;
6152 }
6153 }
6154 }
6155
6156 if (object->shadow == VM_OBJECT_NULL) {
6157 return 0;
6158 }
6159
6160 object_offset += object->vo_shadow_offset;
6161 object = object->shadow;
6162 }
6163
6164 }
6165
6166 /*
6167 * vm_page_validate_cs_fast():
6168 * Performs a few quick checks to determine if the page's code signature
6169 * really needs to be fully validated. It could:
6170 * 1. have been modified (i.e. automatically tainted),
6171 * 2. have already been validated,
6172 * 3. have already been found to be tainted,
6173 * 4. no longer have a backing store.
6174 * Returns FALSE if the page needs to be fully validated.
6175 */
6176 static boolean_t
6177 vm_page_validate_cs_fast(
6178 vm_page_t page)
6179 {
6180 vm_object_t object;
6181
6182 object = VM_PAGE_OBJECT(page);
6183 vm_object_lock_assert_held(object);
6184
6185 if (page->vmp_wpmapped && !page->vmp_cs_tainted) {
6186 /*
6187 * This page was mapped for "write" access sometime in the
6188 * past and could still be modifiable in the future.
6189 * Consider it tainted.
6190 * [ If the page was already found to be "tainted", no
6191 * need to re-validate. ]
6192 */
6193 vm_object_lock_assert_exclusive(object);
6194 page->vmp_cs_validated = TRUE;
6195 page->vmp_cs_tainted = TRUE;
6196 if (cs_debug) {
6197 printf("CODESIGNING: %s: "
6198 "page %p obj %p off 0x%llx "
6199 "was modified\n",
6200 __FUNCTION__,
6201 page, object, page->vmp_offset);
6202 }
6203 vm_cs_validated_dirtied++;
6204 }
6205
6206 if (page->vmp_cs_validated || page->vmp_cs_tainted) {
6207 return TRUE;
6208 }
6209 vm_object_lock_assert_exclusive(object);
6210
6211 #if CHECK_CS_VALIDATION_BITMAP
6212 kern_return_t kr;
6213
6214 kr = vnode_pager_cs_check_validation_bitmap(
6215 object->pager,
6216 page->vmp_offset + object->paging_offset,
6217 CS_BITMAP_CHECK);
6218 if (kr == KERN_SUCCESS) {
6219 page->vmp_cs_validated = TRUE;
6220 page->vmp_cs_tainted = FALSE;
6221 vm_cs_bitmap_validated++;
6222 return TRUE;
6223 }
6224 #endif /* CHECK_CS_VALIDATION_BITMAP */
6225
6226 if (!object->alive || object->terminating || object->pager == NULL) {
6227 /*
6228 * The object is terminating and we don't have its pager
6229 * so we can't validate the data...
6230 */
6231 return TRUE;
6232 }
6233
6234 /* we need to really validate this page */
6235 vm_object_lock_assert_exclusive(object);
6236 return FALSE;
6237 }
6238
6239 void
6240 vm_page_validate_cs_mapped_slow(
6241 vm_page_t page,
6242 const void *kaddr)
6243 {
6244 vm_object_t object;
6245 memory_object_offset_t mo_offset;
6246 memory_object_t pager;
6247 struct vnode *vnode;
6248 boolean_t validated;
6249 unsigned tainted;
6250
6251 assert(page->vmp_busy);
6252 object = VM_PAGE_OBJECT(page);
6253 vm_object_lock_assert_exclusive(object);
6254
6255 vm_cs_validates++;
6256
6257 /*
6258 * Since we get here to validate a page that was brought in by
6259 * the pager, we know that this pager is all setup and ready
6260 * by now.
6261 */
6262 assert(object->code_signed);
6263 assert(!object->internal);
6264 assert(object->pager != NULL);
6265 assert(object->pager_ready);
6266
6267 pager = object->pager;
6268 assert(object->paging_in_progress);
6269 vnode = vnode_pager_lookup_vnode(pager);
6270 mo_offset = page->vmp_offset + object->paging_offset;
6271
6272 /* verify the SHA1 hash for this page */
6273 tainted = 0;
6274 validated = cs_validate_range(vnode,
6275 pager,
6276 mo_offset,
6277 (const void *)((const char *)kaddr),
6278 PAGE_SIZE_64,
6279 &tainted);
6280
6281 if (tainted & CS_VALIDATE_TAINTED) {
6282 page->vmp_cs_tainted = TRUE;
6283 }
6284 if (tainted & CS_VALIDATE_NX) {
6285 page->vmp_cs_nx = TRUE;
6286 }
6287 if (validated) {
6288 page->vmp_cs_validated = TRUE;
6289 }
6290
6291 #if CHECK_CS_VALIDATION_BITMAP
6292 if (page->vmp_cs_validated && !page->vmp_cs_tainted) {
6293 vnode_pager_cs_check_validation_bitmap(object->pager,
6294 mo_offset,
6295 CS_BITMAP_SET);
6296 }
6297 #endif /* CHECK_CS_VALIDATION_BITMAP */
6298 }
6299
6300 void
6301 vm_page_validate_cs_mapped(
6302 vm_page_t page,
6303 const void *kaddr)
6304 {
6305 if (!vm_page_validate_cs_fast(page)) {
6306 vm_page_validate_cs_mapped_slow(page, kaddr);
6307 }
6308 }
6309
6310 void
6311 vm_page_validate_cs(
6312 vm_page_t page)
6313 {
6314 vm_object_t object;
6315 vm_object_offset_t offset;
6316 vm_map_offset_t koffset;
6317 vm_map_size_t ksize;
6318 vm_offset_t kaddr;
6319 kern_return_t kr;
6320 boolean_t busy_page;
6321 boolean_t need_unmap;
6322
6323 object = VM_PAGE_OBJECT(page);
6324 vm_object_lock_assert_held(object);
6325
6326 if (vm_page_validate_cs_fast(page)) {
6327 return;
6328 }
6329 vm_object_lock_assert_exclusive(object);
6330
6331 assert(object->code_signed);
6332 offset = page->vmp_offset;
6333
6334 busy_page = page->vmp_busy;
6335 if (!busy_page) {
6336 /* keep page busy while we map (and unlock) the VM object */
6337 page->vmp_busy = TRUE;
6338 }
6339
6340 /*
6341 * Take a paging reference on the VM object
6342 * to protect it from collapse or bypass,
6343 * and keep it from disappearing too.
6344 */
6345 vm_object_paging_begin(object);
6346
6347 /* map the page in the kernel address space */
6348 ksize = PAGE_SIZE_64;
6349 koffset = 0;
6350 need_unmap = FALSE;
6351 kr = vm_paging_map_object(page,
6352 object,
6353 offset,
6354 VM_PROT_READ,
6355 FALSE, /* can't unlock object ! */
6356 &ksize,
6357 &koffset,
6358 &need_unmap);
6359 if (kr != KERN_SUCCESS) {
6360 panic("%s: could not map page: 0x%x\n", __FUNCTION__, kr);
6361 }
6362 kaddr = CAST_DOWN(vm_offset_t, koffset);
6363
6364 /* validate the mapped page */
6365 vm_page_validate_cs_mapped_slow(page, (const void *) kaddr);
6366
6367 assert(page->vmp_busy);
6368 assert(object == VM_PAGE_OBJECT(page));
6369 vm_object_lock_assert_exclusive(object);
6370
6371 if (!busy_page) {
6372 PAGE_WAKEUP_DONE(page);
6373 }
6374 if (need_unmap) {
6375 /* unmap the map from the kernel address space */
6376 vm_paging_unmap_object(object, koffset, koffset + ksize);
6377 koffset = 0;
6378 ksize = 0;
6379 kaddr = 0;
6380 }
6381 vm_object_paging_end(object);
6382 }
6383
6384 void
6385 vm_page_validate_cs_mapped_chunk(
6386 vm_page_t page,
6387 const void *kaddr,
6388 vm_offset_t chunk_offset,
6389 vm_size_t chunk_size,
6390 boolean_t *validated_p,
6391 unsigned *tainted_p)
6392 {
6393 vm_object_t object;
6394 vm_object_offset_t offset, offset_in_page;
6395 memory_object_t pager;
6396 struct vnode *vnode;
6397 boolean_t validated;
6398 unsigned tainted;
6399
6400 *validated_p = FALSE;
6401 *tainted_p = 0;
6402
6403 assert(page->vmp_busy);
6404 object = VM_PAGE_OBJECT(page);
6405 vm_object_lock_assert_exclusive(object);
6406
6407 assert(object->code_signed);
6408 offset = page->vmp_offset;
6409
6410 if (!object->alive || object->terminating || object->pager == NULL) {
6411 /*
6412 * The object is terminating and we don't have its pager
6413 * so we can't validate the data...
6414 */
6415 return;
6416 }
6417 /*
6418 * Since we get here to validate a page that was brought in by
6419 * the pager, we know that this pager is all setup and ready
6420 * by now.
6421 */
6422 assert(!object->internal);
6423 assert(object->pager != NULL);
6424 assert(object->pager_ready);
6425
6426 pager = object->pager;
6427 assert(object->paging_in_progress);
6428 vnode = vnode_pager_lookup_vnode(pager);
6429
6430 /* verify the signature for this chunk */
6431 offset_in_page = chunk_offset;
6432 assert(offset_in_page < PAGE_SIZE);
6433
6434 tainted = 0;
6435 validated = cs_validate_range(vnode,
6436 pager,
6437 (object->paging_offset +
6438 offset +
6439 offset_in_page),
6440 (const void *)((const char *)kaddr
6441 + offset_in_page),
6442 chunk_size,
6443 &tainted);
6444 if (validated) {
6445 *validated_p = TRUE;
6446 }
6447 if (tainted) {
6448 *tainted_p = tainted;
6449 }
6450 }
6451
6452 static void vm_rtfrecord_lock(void) {
6453 lck_spin_lock(&vm_rtfr_slock);
6454 }
6455
6456 static void vm_rtfrecord_unlock(void) {
6457 lck_spin_unlock(&vm_rtfr_slock);
6458 }
6459
6460 unsigned int vmrtfaultinfo_bufsz(void) {
6461 return (vmrtf_num_records * sizeof(vm_rtfault_record_t));
6462 }
6463
6464 #include <kern/backtrace.h>
6465
6466 static void vm_record_rtfault(thread_t cthread, uint64_t fstart, vm_map_offset_t fault_vaddr, int type_of_fault) {
6467 uint64_t fend = mach_continuous_time();
6468
6469 uint64_t cfpc = 0;
6470 uint64_t ctid = cthread->thread_id;
6471 uint64_t cupid = get_current_unique_pid();
6472
6473 uintptr_t bpc = 0;
6474 uint32_t bfrs = 0;
6475 bool u64 = false;
6476
6477 /* Capture a single-frame backtrace; this extracts just the program
6478 * counter at the point of the fault into "bpc", and should perform no
6479 * further user stack traversals, thus avoiding copyin()s and further
6480 * faults.
6481 */
6482 int btr = backtrace_thread_user(cthread, &bpc, 1U, &bfrs, &u64);
6483
6484 if ((btr == 0) && (bfrs > 0)) {
6485 cfpc = bpc;
6486 }
6487
6488 assert((fstart != 0) && fend >= fstart);
6489 vm_rtfrecord_lock();
6490 assert(vmrtfrs.vmrtfr_curi <= vmrtfrs.vmrtfr_maxi);
6491
6492 vmrtfrs.vmrtf_total++;
6493 vm_rtfault_record_t *cvmr = &vmrtfrs.vm_rtf_records[vmrtfrs.vmrtfr_curi++];
6494
6495 cvmr->rtfabstime = fstart;
6496 cvmr->rtfduration = fend - fstart;
6497 cvmr->rtfaddr = fault_vaddr;
6498 cvmr->rtfpc = cfpc;
6499 cvmr->rtftype = type_of_fault;
6500 cvmr->rtfupid = cupid;
6501 cvmr->rtftid = ctid;
6502
6503 if (vmrtfrs.vmrtfr_curi > vmrtfrs.vmrtfr_maxi) {
6504 vmrtfrs.vmrtfr_curi = 0;
6505 }
6506
6507 vm_rtfrecord_unlock();
6508 }
6509
6510 int vmrtf_extract(uint64_t cupid, __unused boolean_t isroot, int vrecordsz, void *vrecords, int *vmrtfrv) {
6511 vm_rtfault_record_t *cvmrd = vrecords;
6512 size_t residue = vrecordsz;
6513 int numextracted = 0;
6514 boolean_t early_exit = FALSE;
6515
6516 vm_rtfrecord_lock();
6517
6518 for (int vmfi = 0; vmfi <= vmrtfrs.vmrtfr_maxi; vmfi++) {
6519
6520 if (residue < sizeof(vm_rtfault_record_t)) {
6521 early_exit = TRUE;
6522 break;
6523 }
6524
6525 if (vmrtfrs.vm_rtf_records[vmfi].rtfupid != cupid) {
6526 #if DEVELOPMENT || DEBUG
6527 if (isroot == FALSE) {
6528 continue;
6529 }
6530 #else
6531 continue;
6532 #endif /* DEVDEBUG */
6533 }
6534
6535 *cvmrd = vmrtfrs.vm_rtf_records[vmfi];
6536 cvmrd++;
6537 residue -= sizeof(vm_rtfault_record_t);
6538 numextracted++;
6539 }
6540
6541 vm_rtfrecord_unlock();
6542
6543 *vmrtfrv = numextracted;
6544 return (early_exit);
6545 }
mirror of XNU