]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_fault.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3789.31.2.tar.gz
[apple/xnu.git] / osfmk / vm / vm_fault.c
1 /*
2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56 /*
57 */
58 /*
59 * File: vm_fault.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * Page fault handling module.
63 */
64
65 #include <mach_cluster_stats.h>
66 #include <mach_pagemap.h>
67 #include <libkern/OSAtomic.h>
68
69 #include <mach/mach_types.h>
70 #include <mach/kern_return.h>
71 #include <mach/message.h> /* for error codes */
72 #include <mach/vm_param.h>
73 #include <mach/vm_behavior.h>
74 #include <mach/memory_object.h>
75 /* For memory_object_data_{request,unlock} */
76 #include <mach/sdt.h>
77
78 #include <kern/kern_types.h>
79 #include <kern/host_statistics.h>
80 #include <kern/counters.h>
81 #include <kern/task.h>
82 #include <kern/thread.h>
83 #include <kern/sched_prim.h>
84 #include <kern/host.h>
85 #include <kern/xpr.h>
86 #include <kern/mach_param.h>
87 #include <kern/macro_help.h>
88 #include <kern/zalloc.h>
89 #include <kern/misc_protos.h>
90 #include <kern/policy_internal.h>
91
92 #include <vm/vm_compressor.h>
93 #include <vm/vm_compressor_pager.h>
94 #include <vm/vm_fault.h>
95 #include <vm/vm_map.h>
96 #include <vm/vm_object.h>
97 #include <vm/vm_page.h>
98 #include <vm/vm_kern.h>
99 #include <vm/pmap.h>
100 #include <vm/vm_pageout.h>
101 #include <vm/vm_protos.h>
102 #include <vm/vm_external.h>
103 #include <vm/memory_object.h>
104 #include <vm/vm_purgeable_internal.h> /* Needed by some vm_page.h macros */
105 #include <vm/vm_shared_region.h>
106
107 #include <sys/codesign.h>
108 #include <sys/reason.h>
109 #include <sys/signalvar.h>
110
111 #include <libsa/sys/timers.h> /* for struct timespec */
112
113 #define VM_FAULT_CLASSIFY 0
114
115 #define TRACEFAULTPAGE 0 /* (TEST/DEBUG) */
116
117 unsigned int vm_object_pagein_throttle = 16;
118
119 /*
120 * We apply a hard throttle to the demand zero rate of tasks that we believe are running out of control which
121 * kicks in when swap space runs out. 64-bit programs have massive address spaces and can leak enormous amounts
122 * of memory if they're buggy and can run the system completely out of swap space. If this happens, we
123 * impose a hard throttle on them to prevent them from taking the last bit of memory left. This helps
124 * keep the UI active so that the user has a chance to kill the offending task before the system
125 * completely hangs.
126 *
127 * The hard throttle is only applied when the system is nearly completely out of swap space and is only applied
128 * to tasks that appear to be bloated. When swap runs out, any task using more than vm_hard_throttle_threshold
129 * will be throttled. The throttling is done by giving the thread that's trying to demand zero a page a
130 * delay of HARD_THROTTLE_DELAY microseconds before being allowed to try the page fault again.
131 */
132
133 extern void throttle_lowpri_io(int);
134
135 extern struct vnode *vnode_pager_lookup_vnode(memory_object_t);
136
137 uint64_t vm_hard_throttle_threshold;
138
139
140
141 #define NEED_TO_HARD_THROTTLE_THIS_TASK() (vm_wants_task_throttled(current_task()) || \
142 (vm_page_free_count < vm_page_throttle_limit && \
143 proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO) > THROTTLE_LEVEL_THROTTLED))
144
145
146 #define HARD_THROTTLE_DELAY 5000 /* 5000 us == 5 ms */
147 #define SOFT_THROTTLE_DELAY 200 /* 200 us == .2 ms */
148
149 #define VM_PAGE_CREATION_THROTTLE_PERIOD_SECS 6
150 #define VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC 20000
151
152
153 boolean_t current_thread_aborted(void);
154
155 /* Forward declarations of internal routines. */
156 static kern_return_t vm_fault_wire_fast(
157 vm_map_t map,
158 vm_map_offset_t va,
159 vm_prot_t prot,
160 vm_map_entry_t entry,
161 pmap_t pmap,
162 vm_map_offset_t pmap_addr,
163 ppnum_t *physpage_p);
164
165 static kern_return_t vm_fault_internal(
166 vm_map_t map,
167 vm_map_offset_t vaddr,
168 vm_prot_t caller_prot,
169 boolean_t change_wiring,
170 int interruptible,
171 pmap_t pmap,
172 vm_map_offset_t pmap_addr,
173 ppnum_t *physpage_p);
174
175 static void vm_fault_copy_cleanup(
176 vm_page_t page,
177 vm_page_t top_page);
178
179 static void vm_fault_copy_dst_cleanup(
180 vm_page_t page);
181
182 #if VM_FAULT_CLASSIFY
183 extern void vm_fault_classify(vm_object_t object,
184 vm_object_offset_t offset,
185 vm_prot_t fault_type);
186
187 extern void vm_fault_classify_init(void);
188 #endif
189
190 unsigned long vm_pmap_enter_blocked = 0;
191 unsigned long vm_pmap_enter_retried = 0;
192
193 unsigned long vm_cs_validates = 0;
194 unsigned long vm_cs_revalidates = 0;
195 unsigned long vm_cs_query_modified = 0;
196 unsigned long vm_cs_validated_dirtied = 0;
197 unsigned long vm_cs_bitmap_validated = 0;
198
199 void vm_pre_fault(vm_map_offset_t);
200
201 extern char *kdp_compressor_decompressed_page;
202 extern addr64_t kdp_compressor_decompressed_page_paddr;
203 extern ppnum_t kdp_compressor_decompressed_page_ppnum;
204
205 /*
206 * Routine: vm_fault_init
207 * Purpose:
208 * Initialize our private data structures.
209 */
210 void
211 vm_fault_init(void)
212 {
213 int i, vm_compressor_temp;
214 boolean_t need_default_val = TRUE;
215 /*
216 * Choose a value for the hard throttle threshold based on the amount of ram. The threshold is
217 * computed as a percentage of available memory, and the percentage used is scaled inversely with
218 * the amount of memory. The percentage runs between 10% and 35%. We use 35% for small memory systems
219 * and reduce the value down to 10% for very large memory configurations. This helps give us a
220 * definition of a memory hog that makes more sense relative to the amount of ram in the machine.
221 * The formula here simply uses the number of gigabytes of ram to adjust the percentage.
222 */
223
224 vm_hard_throttle_threshold = sane_size * (35 - MIN((int)(sane_size / (1024*1024*1024)), 25)) / 100;
225
226 /*
227 * Configure compressed pager behavior. A boot arg takes precedence over a device tree entry.
228 */
229
230 if (PE_parse_boot_argn("vm_compressor", &vm_compressor_temp, sizeof (vm_compressor_temp))) {
231 for ( i = 0; i < VM_PAGER_MAX_MODES; i++) {
232 if (vm_compressor_temp > 0 &&
233 ((vm_compressor_temp & ( 1 << i)) == vm_compressor_temp)) {
234 need_default_val = FALSE;
235 vm_compressor_mode = vm_compressor_temp;
236 break;
237 }
238 }
239 if (need_default_val)
240 printf("Ignoring \"vm_compressor\" boot arg %d\n", vm_compressor_temp);
241 }
242 if (need_default_val) {
243 /* If no boot arg or incorrect boot arg, try device tree. */
244 PE_get_default("kern.vm_compressor", &vm_compressor_mode, sizeof(vm_compressor_mode));
245 }
246 PE_parse_boot_argn("vm_compressor_threads", &vm_compressor_thread_count, sizeof (vm_compressor_thread_count));
247
248 if (PE_parse_boot_argn("vm_compressor_immediate", &vm_compressor_temp, sizeof (vm_compressor_temp)))
249 vm_compressor_immediate_preferred_override = TRUE;
250 else {
251 if (PE_get_default("kern.vm_compressor_immediate", &vm_compressor_temp, sizeof(vm_compressor_temp)))
252 vm_compressor_immediate_preferred_override = TRUE;
253 }
254 if (vm_compressor_immediate_preferred_override == TRUE) {
255 if (vm_compressor_temp)
256 vm_compressor_immediate_preferred = TRUE;
257 else
258 vm_compressor_immediate_preferred = FALSE;
259 }
260 printf("\"vm_compressor_mode\" is %d\n", vm_compressor_mode);
261 }
262
263 /*
264 * Routine: vm_fault_cleanup
265 * Purpose:
266 * Clean up the result of vm_fault_page.
267 * Results:
268 * The paging reference for "object" is released.
269 * "object" is unlocked.
270 * If "top_page" is not null, "top_page" is
271 * freed and the paging reference for the object
272 * containing it is released.
273 *
274 * In/out conditions:
275 * "object" must be locked.
276 */
277 void
278 vm_fault_cleanup(
279 vm_object_t object,
280 vm_page_t top_page)
281 {
282 vm_object_paging_end(object);
283 vm_object_unlock(object);
284
285 if (top_page != VM_PAGE_NULL) {
286 object = VM_PAGE_OBJECT(top_page);
287
288 vm_object_lock(object);
289 VM_PAGE_FREE(top_page);
290 vm_object_paging_end(object);
291 vm_object_unlock(object);
292 }
293 }
294
295 #if MACH_CLUSTER_STATS
296 #define MAXCLUSTERPAGES 16
297 struct {
298 unsigned long pages_in_cluster;
299 unsigned long pages_at_higher_offsets;
300 unsigned long pages_at_lower_offsets;
301 } cluster_stats_in[MAXCLUSTERPAGES];
302 #define CLUSTER_STAT(clause) clause
303 #define CLUSTER_STAT_HIGHER(x) \
304 ((cluster_stats_in[(x)].pages_at_higher_offsets)++)
305 #define CLUSTER_STAT_LOWER(x) \
306 ((cluster_stats_in[(x)].pages_at_lower_offsets)++)
307 #define CLUSTER_STAT_CLUSTER(x) \
308 ((cluster_stats_in[(x)].pages_in_cluster)++)
309 #else /* MACH_CLUSTER_STATS */
310 #define CLUSTER_STAT(clause)
311 #endif /* MACH_CLUSTER_STATS */
312
313 #define ALIGNED(x) (((x) & (PAGE_SIZE_64 - 1)) == 0)
314
315
316 boolean_t vm_page_deactivate_behind = TRUE;
317 /*
318 * default sizes given VM_BEHAVIOR_DEFAULT reference behavior
319 */
320 #define VM_DEFAULT_DEACTIVATE_BEHIND_WINDOW 128
321 #define VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER 16 /* don't make this too big... */
322 /* we use it to size an array on the stack */
323
324 int vm_default_behind = VM_DEFAULT_DEACTIVATE_BEHIND_WINDOW;
325
326 #define MAX_SEQUENTIAL_RUN (1024 * 1024 * 1024)
327
328 /*
329 * vm_page_is_sequential
330 *
331 * Determine if sequential access is in progress
332 * in accordance with the behavior specified.
333 * Update state to indicate current access pattern.
334 *
335 * object must have at least the shared lock held
336 */
337 static
338 void
339 vm_fault_is_sequential(
340 vm_object_t object,
341 vm_object_offset_t offset,
342 vm_behavior_t behavior)
343 {
344 vm_object_offset_t last_alloc;
345 int sequential;
346 int orig_sequential;
347
348 last_alloc = object->last_alloc;
349 sequential = object->sequential;
350 orig_sequential = sequential;
351
352 switch (behavior) {
353 case VM_BEHAVIOR_RANDOM:
354 /*
355 * reset indicator of sequential behavior
356 */
357 sequential = 0;
358 break;
359
360 case VM_BEHAVIOR_SEQUENTIAL:
361 if (offset && last_alloc == offset - PAGE_SIZE_64) {
362 /*
363 * advance indicator of sequential behavior
364 */
365 if (sequential < MAX_SEQUENTIAL_RUN)
366 sequential += PAGE_SIZE;
367 } else {
368 /*
369 * reset indicator of sequential behavior
370 */
371 sequential = 0;
372 }
373 break;
374
375 case VM_BEHAVIOR_RSEQNTL:
376 if (last_alloc && last_alloc == offset + PAGE_SIZE_64) {
377 /*
378 * advance indicator of sequential behavior
379 */
380 if (sequential > -MAX_SEQUENTIAL_RUN)
381 sequential -= PAGE_SIZE;
382 } else {
383 /*
384 * reset indicator of sequential behavior
385 */
386 sequential = 0;
387 }
388 break;
389
390 case VM_BEHAVIOR_DEFAULT:
391 default:
392 if (offset && last_alloc == (offset - PAGE_SIZE_64)) {
393 /*
394 * advance indicator of sequential behavior
395 */
396 if (sequential < 0)
397 sequential = 0;
398 if (sequential < MAX_SEQUENTIAL_RUN)
399 sequential += PAGE_SIZE;
400
401 } else if (last_alloc && last_alloc == (offset + PAGE_SIZE_64)) {
402 /*
403 * advance indicator of sequential behavior
404 */
405 if (sequential > 0)
406 sequential = 0;
407 if (sequential > -MAX_SEQUENTIAL_RUN)
408 sequential -= PAGE_SIZE;
409 } else {
410 /*
411 * reset indicator of sequential behavior
412 */
413 sequential = 0;
414 }
415 break;
416 }
417 if (sequential != orig_sequential) {
418 if (!OSCompareAndSwap(orig_sequential, sequential, (UInt32 *)&object->sequential)) {
419 /*
420 * if someone else has already updated object->sequential
421 * don't bother trying to update it or object->last_alloc
422 */
423 return;
424 }
425 }
426 /*
427 * I'd like to do this with a OSCompareAndSwap64, but that
428 * doesn't exist for PPC... however, it shouldn't matter
429 * that much... last_alloc is maintained so that we can determine
430 * if a sequential access pattern is taking place... if only
431 * one thread is banging on this object, no problem with the unprotected
432 * update... if 2 or more threads are banging away, we run the risk of
433 * someone seeing a mangled update... however, in the face of multiple
434 * accesses, no sequential access pattern can develop anyway, so we
435 * haven't lost any real info.
436 */
437 object->last_alloc = offset;
438 }
439
440
441 int vm_page_deactivate_behind_count = 0;
442
443 /*
444 * vm_page_deactivate_behind
445 *
446 * Determine if sequential access is in progress
447 * in accordance with the behavior specified. If
448 * so, compute a potential page to deactivate and
449 * deactivate it.
450 *
451 * object must be locked.
452 *
453 * return TRUE if we actually deactivate a page
454 */
455 static
456 boolean_t
457 vm_fault_deactivate_behind(
458 vm_object_t object,
459 vm_object_offset_t offset,
460 vm_behavior_t behavior)
461 {
462 int n;
463 int pages_in_run = 0;
464 int max_pages_in_run = 0;
465 int sequential_run;
466 int sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
467 vm_object_offset_t run_offset = 0;
468 vm_object_offset_t pg_offset = 0;
469 vm_page_t m;
470 vm_page_t page_run[VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER];
471
472 pages_in_run = 0;
473 #if TRACEFAULTPAGE
474 dbgTrace(0xBEEF0018, (unsigned int) object, (unsigned int) vm_fault_deactivate_behind); /* (TEST/DEBUG) */
475 #endif
476
477 if (object == kernel_object || vm_page_deactivate_behind == FALSE) {
478 /*
479 * Do not deactivate pages from the kernel object: they
480 * are not intended to become pageable.
481 * or we've disabled the deactivate behind mechanism
482 */
483 return FALSE;
484 }
485 if ((sequential_run = object->sequential)) {
486 if (sequential_run < 0) {
487 sequential_behavior = VM_BEHAVIOR_RSEQNTL;
488 sequential_run = 0 - sequential_run;
489 } else {
490 sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
491 }
492 }
493 switch (behavior) {
494 case VM_BEHAVIOR_RANDOM:
495 break;
496 case VM_BEHAVIOR_SEQUENTIAL:
497 if (sequential_run >= (int)PAGE_SIZE) {
498 run_offset = 0 - PAGE_SIZE_64;
499 max_pages_in_run = 1;
500 }
501 break;
502 case VM_BEHAVIOR_RSEQNTL:
503 if (sequential_run >= (int)PAGE_SIZE) {
504 run_offset = PAGE_SIZE_64;
505 max_pages_in_run = 1;
506 }
507 break;
508 case VM_BEHAVIOR_DEFAULT:
509 default:
510 { vm_object_offset_t behind = vm_default_behind * PAGE_SIZE_64;
511
512 /*
513 * determine if the run of sequential accesss has been
514 * long enough on an object with default access behavior
515 * to consider it for deactivation
516 */
517 if ((uint64_t)sequential_run >= behind && (sequential_run % (VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER * PAGE_SIZE)) == 0) {
518 /*
519 * the comparisons between offset and behind are done
520 * in this kind of odd fashion in order to prevent wrap around
521 * at the end points
522 */
523 if (sequential_behavior == VM_BEHAVIOR_SEQUENTIAL) {
524 if (offset >= behind) {
525 run_offset = 0 - behind;
526 pg_offset = PAGE_SIZE_64;
527 max_pages_in_run = VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER;
528 }
529 } else {
530 if (offset < -behind) {
531 run_offset = behind;
532 pg_offset = 0 - PAGE_SIZE_64;
533 max_pages_in_run = VM_DEFAULT_DEACTIVATE_BEHIND_CLUSTER;
534 }
535 }
536 }
537 break;
538 }
539 }
540 for (n = 0; n < max_pages_in_run; n++) {
541 m = vm_page_lookup(object, offset + run_offset + (n * pg_offset));
542
543 if (m && !m->laundry && !m->busy && !m->no_cache && (m->vm_page_q_state != VM_PAGE_ON_THROTTLED_Q) && !m->fictitious && !m->absent) {
544 page_run[pages_in_run++] = m;
545
546 /*
547 * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise...
548 *
549 * a TLB flush isn't really needed here since at worst we'll miss the reference bit being
550 * updated in the PTE if a remote processor still has this mapping cached in its TLB when the
551 * new reference happens. If no futher references happen on the page after that remote TLB flushes
552 * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue
553 * by pageout_scan, which is just fine since the last reference would have happened quite far
554 * in the past (TLB caches don't hang around for very long), and of course could just as easily
555 * have happened before we did the deactivate_behind.
556 */
557 pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
558 }
559 }
560 if (pages_in_run) {
561 vm_page_lockspin_queues();
562
563 for (n = 0; n < pages_in_run; n++) {
564
565 m = page_run[n];
566
567 vm_page_deactivate_internal(m, FALSE);
568
569 vm_page_deactivate_behind_count++;
570 #if TRACEFAULTPAGE
571 dbgTrace(0xBEEF0019, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
572 #endif
573 }
574 vm_page_unlock_queues();
575
576 return TRUE;
577 }
578 return FALSE;
579 }
580
581
582 #if (DEVELOPMENT || DEBUG)
583 uint32_t vm_page_creation_throttled_hard = 0;
584 uint32_t vm_page_creation_throttled_soft = 0;
585 uint64_t vm_page_creation_throttle_avoided = 0;
586 #endif /* DEVELOPMENT || DEBUG */
587
588 static int
589 vm_page_throttled(boolean_t page_kept)
590 {
591 clock_sec_t elapsed_sec;
592 clock_sec_t tv_sec;
593 clock_usec_t tv_usec;
594
595 thread_t thread = current_thread();
596
597 if (thread->options & TH_OPT_VMPRIV)
598 return (0);
599
600 if (thread->t_page_creation_throttled) {
601 thread->t_page_creation_throttled = 0;
602
603 if (page_kept == FALSE)
604 goto no_throttle;
605 }
606 if (NEED_TO_HARD_THROTTLE_THIS_TASK()) {
607 #if (DEVELOPMENT || DEBUG)
608 thread->t_page_creation_throttled_hard++;
609 OSAddAtomic(1, &vm_page_creation_throttled_hard);
610 #endif /* DEVELOPMENT || DEBUG */
611 return (HARD_THROTTLE_DELAY);
612 }
613
614 if ((vm_page_free_count < vm_page_throttle_limit || (VM_CONFIG_COMPRESSOR_IS_PRESENT && SWAPPER_NEEDS_TO_UNTHROTTLE())) &&
615 thread->t_page_creation_count > (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS * VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC)) {
616
617 if (vm_page_free_wanted == 0 && vm_page_free_wanted_privileged == 0) {
618 #if (DEVELOPMENT || DEBUG)
619 OSAddAtomic64(1, &vm_page_creation_throttle_avoided);
620 #endif
621 goto no_throttle;
622 }
623 clock_get_system_microtime(&tv_sec, &tv_usec);
624
625 elapsed_sec = tv_sec - thread->t_page_creation_time;
626
627 if (elapsed_sec <= VM_PAGE_CREATION_THROTTLE_PERIOD_SECS ||
628 (thread->t_page_creation_count / elapsed_sec) >= VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC) {
629
630 if (elapsed_sec >= (3 * VM_PAGE_CREATION_THROTTLE_PERIOD_SECS)) {
631 /*
632 * we'll reset our stats to give a well behaved app
633 * that was unlucky enough to accumulate a bunch of pages
634 * over a long period of time a chance to get out of
635 * the throttled state... we reset the counter and timestamp
636 * so that if it stays under the rate limit for the next second
637 * it will be back in our good graces... if it exceeds it, it
638 * will remain in the throttled state
639 */
640 thread->t_page_creation_time = tv_sec;
641 thread->t_page_creation_count = VM_PAGE_CREATION_THROTTLE_RATE_PER_SEC * (VM_PAGE_CREATION_THROTTLE_PERIOD_SECS - 1);
642 }
643 ++vm_page_throttle_count;
644
645 thread->t_page_creation_throttled = 1;
646
647 if (VM_CONFIG_COMPRESSOR_IS_PRESENT && HARD_THROTTLE_LIMIT_REACHED()) {
648 #if (DEVELOPMENT || DEBUG)
649 thread->t_page_creation_throttled_hard++;
650 OSAddAtomic(1, &vm_page_creation_throttled_hard);
651 #endif /* DEVELOPMENT || DEBUG */
652 return (HARD_THROTTLE_DELAY);
653 } else {
654 #if (DEVELOPMENT || DEBUG)
655 thread->t_page_creation_throttled_soft++;
656 OSAddAtomic(1, &vm_page_creation_throttled_soft);
657 #endif /* DEVELOPMENT || DEBUG */
658 return (SOFT_THROTTLE_DELAY);
659 }
660 }
661 thread->t_page_creation_time = tv_sec;
662 thread->t_page_creation_count = 0;
663 }
664 no_throttle:
665 thread->t_page_creation_count++;
666
667 return (0);
668 }
669
670
671 /*
672 * check for various conditions that would
673 * prevent us from creating a ZF page...
674 * cleanup is based on being called from vm_fault_page
675 *
676 * object must be locked
677 * object == m->object
678 */
679 static vm_fault_return_t
680 vm_fault_check(vm_object_t object, vm_page_t m, vm_page_t first_m, boolean_t interruptible_state, boolean_t page_throttle)
681 {
682 int throttle_delay;
683
684 if (object->shadow_severed ||
685 VM_OBJECT_PURGEABLE_FAULT_ERROR(object)) {
686 /*
687 * Either:
688 * 1. the shadow chain was severed,
689 * 2. the purgeable object is volatile or empty and is marked
690 * to fault on access while volatile.
691 * Just have to return an error at this point
692 */
693 if (m != VM_PAGE_NULL)
694 VM_PAGE_FREE(m);
695 vm_fault_cleanup(object, first_m);
696
697 thread_interrupt_level(interruptible_state);
698
699 return (VM_FAULT_MEMORY_ERROR);
700 }
701 if (vm_backing_store_low) {
702 /*
703 * are we protecting the system from
704 * backing store exhaustion. If so
705 * sleep unless we are privileged.
706 */
707 if (!(current_task()->priv_flags & VM_BACKING_STORE_PRIV)) {
708
709 if (m != VM_PAGE_NULL)
710 VM_PAGE_FREE(m);
711 vm_fault_cleanup(object, first_m);
712
713 assert_wait((event_t)&vm_backing_store_low, THREAD_UNINT);
714
715 thread_block(THREAD_CONTINUE_NULL);
716 thread_interrupt_level(interruptible_state);
717
718 return (VM_FAULT_RETRY);
719 }
720 }
721 if (page_throttle == TRUE) {
722 if ((throttle_delay = vm_page_throttled(FALSE))) {
723 /*
724 * we're throttling zero-fills...
725 * treat this as if we couldn't grab a page
726 */
727 if (m != VM_PAGE_NULL)
728 VM_PAGE_FREE(m);
729 vm_fault_cleanup(object, first_m);
730
731 VM_DEBUG_EVENT(vmf_check_zfdelay, VMF_CHECK_ZFDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
732
733 delay(throttle_delay);
734
735 if (current_thread_aborted()) {
736 thread_interrupt_level(interruptible_state);
737 return VM_FAULT_INTERRUPTED;
738 }
739 thread_interrupt_level(interruptible_state);
740
741 return (VM_FAULT_MEMORY_SHORTAGE);
742 }
743 }
744 return (VM_FAULT_SUCCESS);
745 }
746
747
748 /*
749 * do the work to zero fill a page and
750 * inject it into the correct paging queue
751 *
752 * m->object must be locked
753 * page queue lock must NOT be held
754 */
755 static int
756 vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill)
757 {
758 int my_fault = DBG_ZERO_FILL_FAULT;
759 vm_object_t object;
760
761 object = VM_PAGE_OBJECT(m);
762
763 /*
764 * This is is a zero-fill page fault...
765 *
766 * Checking the page lock is a waste of
767 * time; this page was absent, so
768 * it can't be page locked by a pager.
769 *
770 * we also consider it undefined
771 * with respect to instruction
772 * execution. i.e. it is the responsibility
773 * of higher layers to call for an instruction
774 * sync after changing the contents and before
775 * sending a program into this area. We
776 * choose this approach for performance
777 */
778 m->pmapped = TRUE;
779
780 m->cs_validated = FALSE;
781 m->cs_tainted = FALSE;
782 m->cs_nx = FALSE;
783
784 if (no_zero_fill == TRUE) {
785 my_fault = DBG_NZF_PAGE_FAULT;
786
787 if (m->absent && m->busy)
788 return (my_fault);
789 } else {
790 vm_page_zero_fill(m);
791
792 VM_STAT_INCR(zero_fill_count);
793 DTRACE_VM2(zfod, int, 1, (uint64_t *), NULL);
794 }
795 assert(!m->laundry);
796 assert(object != kernel_object);
797 //assert(m->pageq.next == 0 && m->pageq.prev == 0);
798
799 if (!VM_DYNAMIC_PAGING_ENABLED() &&
800 (object->purgable == VM_PURGABLE_DENY ||
801 object->purgable == VM_PURGABLE_NONVOLATILE ||
802 object->purgable == VM_PURGABLE_VOLATILE )) {
803
804 vm_page_lockspin_queues();
805
806 if (!VM_DYNAMIC_PAGING_ENABLED()) {
807 assert(!VM_PAGE_WIRED(m));
808
809 /*
810 * can't be on the pageout queue since we don't
811 * have a pager to try and clean to
812 */
813 vm_page_queues_remove(m, TRUE);
814 vm_page_check_pageable_safe(m);
815 vm_page_queue_enter(&vm_page_queue_throttled, m, vm_page_t, pageq);
816 m->vm_page_q_state = VM_PAGE_ON_THROTTLED_Q;
817 vm_page_throttled_count++;
818 }
819 vm_page_unlock_queues();
820 }
821 return (my_fault);
822 }
823
824
825 /*
826 * Routine: vm_fault_page
827 * Purpose:
828 * Find the resident page for the virtual memory
829 * specified by the given virtual memory object
830 * and offset.
831 * Additional arguments:
832 * The required permissions for the page is given
833 * in "fault_type". Desired permissions are included
834 * in "protection".
835 * fault_info is passed along to determine pagein cluster
836 * limits... it contains the expected reference pattern,
837 * cluster size if available, etc...
838 *
839 * If the desired page is known to be resident (for
840 * example, because it was previously wired down), asserting
841 * the "unwiring" parameter will speed the search.
842 *
843 * If the operation can be interrupted (by thread_abort
844 * or thread_terminate), then the "interruptible"
845 * parameter should be asserted.
846 *
847 * Results:
848 * The page containing the proper data is returned
849 * in "result_page".
850 *
851 * In/out conditions:
852 * The source object must be locked and referenced,
853 * and must donate one paging reference. The reference
854 * is not affected. The paging reference and lock are
855 * consumed.
856 *
857 * If the call succeeds, the object in which "result_page"
858 * resides is left locked and holding a paging reference.
859 * If this is not the original object, a busy page in the
860 * original object is returned in "top_page", to prevent other
861 * callers from pursuing this same data, along with a paging
862 * reference for the original object. The "top_page" should
863 * be destroyed when this guarantee is no longer required.
864 * The "result_page" is also left busy. It is not removed
865 * from the pageout queues.
866 * Special Case:
867 * A return value of VM_FAULT_SUCCESS_NO_PAGE means that the
868 * fault succeeded but there's no VM page (i.e. the VM object
869 * does not actually hold VM pages, but device memory or
870 * large pages). The object is still locked and we still hold a
871 * paging_in_progress reference.
872 */
873 unsigned int vm_fault_page_blocked_access = 0;
874 unsigned int vm_fault_page_forced_retry = 0;
875
876 vm_fault_return_t
877 vm_fault_page(
878 /* Arguments: */
879 vm_object_t first_object, /* Object to begin search */
880 vm_object_offset_t first_offset, /* Offset into object */
881 vm_prot_t fault_type, /* What access is requested */
882 boolean_t must_be_resident,/* Must page be resident? */
883 boolean_t caller_lookup, /* caller looked up page */
884 /* Modifies in place: */
885 vm_prot_t *protection, /* Protection for mapping */
886 vm_page_t *result_page, /* Page found, if successful */
887 /* Returns: */
888 vm_page_t *top_page, /* Page in top object, if
889 * not result_page. */
890 int *type_of_fault, /* if non-null, fill in with type of fault
891 * COW, zero-fill, etc... returned in trace point */
892 /* More arguments: */
893 kern_return_t *error_code, /* code if page is in error */
894 boolean_t no_zero_fill, /* don't zero fill absent pages */
895 boolean_t data_supply, /* treat as data_supply if
896 * it is a write fault and a full
897 * page is provided */
898 vm_object_fault_info_t fault_info)
899 {
900 vm_page_t m;
901 vm_object_t object;
902 vm_object_offset_t offset;
903 vm_page_t first_m;
904 vm_object_t next_object;
905 vm_object_t copy_object;
906 boolean_t look_for_page;
907 boolean_t force_fault_retry = FALSE;
908 vm_prot_t access_required = fault_type;
909 vm_prot_t wants_copy_flag;
910 CLUSTER_STAT(int pages_at_higher_offsets;)
911 CLUSTER_STAT(int pages_at_lower_offsets;)
912 kern_return_t wait_result;
913 boolean_t interruptible_state;
914 boolean_t data_already_requested = FALSE;
915 vm_behavior_t orig_behavior;
916 vm_size_t orig_cluster_size;
917 vm_fault_return_t error;
918 int my_fault;
919 uint32_t try_failed_count;
920 int interruptible; /* how may fault be interrupted? */
921 int external_state = VM_EXTERNAL_STATE_UNKNOWN;
922 memory_object_t pager;
923 vm_fault_return_t retval;
924 int grab_options;
925
926 /*
927 * MUST_ASK_PAGER() evaluates to TRUE if the page specified by object/offset is
928 * marked as paged out in the compressor pager or the pager doesn't exist.
929 * Note also that if the pager for an internal object
930 * has not been created, the pager is not invoked regardless of the value
931 * of MUST_ASK_PAGER().
932 *
933 * PAGED_OUT() evaluates to TRUE if the page specified by the object/offset
934 * is marked as paged out in the compressor pager.
935 * PAGED_OUT() is used to determine if a page has already been pushed
936 * into a copy object in order to avoid a redundant page out operation.
937 */
938 #define MUST_ASK_PAGER(o, f, s) \
939 ((s = VM_COMPRESSOR_PAGER_STATE_GET((o), (f))) != VM_EXTERNAL_STATE_ABSENT)
940
941 #define PAGED_OUT(o, f) \
942 (VM_COMPRESSOR_PAGER_STATE_GET((o), (f)) == VM_EXTERNAL_STATE_EXISTS)
943
944 /*
945 * Recovery actions
946 */
947 #define RELEASE_PAGE(m) \
948 MACRO_BEGIN \
949 PAGE_WAKEUP_DONE(m); \
950 if ( !VM_PAGE_PAGEABLE(m)) { \
951 vm_page_lockspin_queues(); \
952 if ( !VM_PAGE_PAGEABLE(m)) { \
953 if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) \
954 vm_page_deactivate(m); \
955 else \
956 vm_page_activate(m); \
957 } \
958 vm_page_unlock_queues(); \
959 } \
960 MACRO_END
961
962 #if TRACEFAULTPAGE
963 dbgTrace(0xBEEF0002, (unsigned int) first_object, (unsigned int) first_offset); /* (TEST/DEBUG) */
964 #endif
965
966 interruptible = fault_info->interruptible;
967 interruptible_state = thread_interrupt_level(interruptible);
968
969 /*
970 * INVARIANTS (through entire routine):
971 *
972 * 1) At all times, we must either have the object
973 * lock or a busy page in some object to prevent
974 * some other thread from trying to bring in
975 * the same page.
976 *
977 * Note that we cannot hold any locks during the
978 * pager access or when waiting for memory, so
979 * we use a busy page then.
980 *
981 * 2) To prevent another thread from racing us down the
982 * shadow chain and entering a new page in the top
983 * object before we do, we must keep a busy page in
984 * the top object while following the shadow chain.
985 *
986 * 3) We must increment paging_in_progress on any object
987 * for which we have a busy page before dropping
988 * the object lock
989 *
990 * 4) We leave busy pages on the pageout queues.
991 * If the pageout daemon comes across a busy page,
992 * it will remove the page from the pageout queues.
993 */
994
995 object = first_object;
996 offset = first_offset;
997 first_m = VM_PAGE_NULL;
998 access_required = fault_type;
999
1000
1001 XPR(XPR_VM_FAULT,
1002 "vm_f_page: obj 0x%X, offset 0x%X, type %d, prot %d\n",
1003 object, offset, fault_type, *protection, 0);
1004
1005 /*
1006 * default type of fault
1007 */
1008 my_fault = DBG_CACHE_HIT_FAULT;
1009
1010 while (TRUE) {
1011 #if TRACEFAULTPAGE
1012 dbgTrace(0xBEEF0003, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */
1013 #endif
1014
1015 grab_options = 0;
1016 #if CONFIG_SECLUDED_MEMORY
1017 if (object->can_grab_secluded) {
1018 grab_options |= VM_PAGE_GRAB_SECLUDED;
1019 }
1020 #endif /* CONFIG_SECLUDED_MEMORY */
1021
1022 if (!object->alive) {
1023 /*
1024 * object is no longer valid
1025 * clean up and return error
1026 */
1027 vm_fault_cleanup(object, first_m);
1028 thread_interrupt_level(interruptible_state);
1029
1030 return (VM_FAULT_MEMORY_ERROR);
1031 }
1032
1033 if (!object->pager_created && object->phys_contiguous) {
1034 /*
1035 * A physically-contiguous object without a pager:
1036 * must be a "large page" object. We do not deal
1037 * with VM pages for this object.
1038 */
1039 caller_lookup = FALSE;
1040 m = VM_PAGE_NULL;
1041 goto phys_contig_object;
1042 }
1043
1044 if (object->blocked_access) {
1045 /*
1046 * Access to this VM object has been blocked.
1047 * Replace our "paging_in_progress" reference with
1048 * a "activity_in_progress" reference and wait for
1049 * access to be unblocked.
1050 */
1051 caller_lookup = FALSE; /* no longer valid after sleep */
1052 vm_object_activity_begin(object);
1053 vm_object_paging_end(object);
1054 while (object->blocked_access) {
1055 vm_object_sleep(object,
1056 VM_OBJECT_EVENT_UNBLOCKED,
1057 THREAD_UNINT);
1058 }
1059 vm_fault_page_blocked_access++;
1060 vm_object_paging_begin(object);
1061 vm_object_activity_end(object);
1062 }
1063
1064 /*
1065 * See whether the page at 'offset' is resident
1066 */
1067 if (caller_lookup == TRUE) {
1068 /*
1069 * The caller has already looked up the page
1070 * and gave us the result in "result_page".
1071 * We can use this for the first lookup but
1072 * it loses its validity as soon as we unlock
1073 * the object.
1074 */
1075 m = *result_page;
1076 caller_lookup = FALSE; /* no longer valid after that */
1077 } else {
1078 m = vm_page_lookup(object, offset);
1079 }
1080 #if TRACEFAULTPAGE
1081 dbgTrace(0xBEEF0004, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
1082 #endif
1083 if (m != VM_PAGE_NULL) {
1084
1085 if (m->busy) {
1086 /*
1087 * The page is being brought in,
1088 * wait for it and then retry.
1089 */
1090 #if TRACEFAULTPAGE
1091 dbgTrace(0xBEEF0005, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1092 #endif
1093 wait_result = PAGE_SLEEP(object, m, interruptible);
1094
1095 XPR(XPR_VM_FAULT,
1096 "vm_f_page: block busy obj 0x%X, offset 0x%X, page 0x%X\n",
1097 object, offset,
1098 m, 0, 0);
1099 counter(c_vm_fault_page_block_busy_kernel++);
1100
1101 if (wait_result != THREAD_AWAKENED) {
1102 vm_fault_cleanup(object, first_m);
1103 thread_interrupt_level(interruptible_state);
1104
1105 if (wait_result == THREAD_RESTART)
1106 return (VM_FAULT_RETRY);
1107 else
1108 return (VM_FAULT_INTERRUPTED);
1109 }
1110 continue;
1111 }
1112 if (m->laundry) {
1113 m->free_when_done = FALSE;
1114
1115 if (!m->cleaning)
1116 vm_pageout_steal_laundry(m, FALSE);
1117 }
1118 if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
1119 /*
1120 * Guard page: off limits !
1121 */
1122 if (fault_type == VM_PROT_NONE) {
1123 /*
1124 * The fault is not requesting any
1125 * access to the guard page, so it must
1126 * be just to wire or unwire it.
1127 * Let's pretend it succeeded...
1128 */
1129 m->busy = TRUE;
1130 *result_page = m;
1131 assert(first_m == VM_PAGE_NULL);
1132 *top_page = first_m;
1133 if (type_of_fault)
1134 *type_of_fault = DBG_GUARD_FAULT;
1135 thread_interrupt_level(interruptible_state);
1136 return VM_FAULT_SUCCESS;
1137 } else {
1138 /*
1139 * The fault requests access to the
1140 * guard page: let's deny that !
1141 */
1142 vm_fault_cleanup(object, first_m);
1143 thread_interrupt_level(interruptible_state);
1144 return VM_FAULT_MEMORY_ERROR;
1145 }
1146 }
1147
1148 if (m->error) {
1149 /*
1150 * The page is in error, give up now.
1151 */
1152 #if TRACEFAULTPAGE
1153 dbgTrace(0xBEEF0006, (unsigned int) m, (unsigned int) error_code); /* (TEST/DEBUG) */
1154 #endif
1155 if (error_code)
1156 *error_code = KERN_MEMORY_ERROR;
1157 VM_PAGE_FREE(m);
1158
1159 vm_fault_cleanup(object, first_m);
1160 thread_interrupt_level(interruptible_state);
1161
1162 return (VM_FAULT_MEMORY_ERROR);
1163 }
1164 if (m->restart) {
1165 /*
1166 * The pager wants us to restart
1167 * at the top of the chain,
1168 * typically because it has moved the
1169 * page to another pager, then do so.
1170 */
1171 #if TRACEFAULTPAGE
1172 dbgTrace(0xBEEF0007, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1173 #endif
1174 VM_PAGE_FREE(m);
1175
1176 vm_fault_cleanup(object, first_m);
1177 thread_interrupt_level(interruptible_state);
1178
1179 return (VM_FAULT_RETRY);
1180 }
1181 if (m->absent) {
1182 /*
1183 * The page isn't busy, but is absent,
1184 * therefore it's deemed "unavailable".
1185 *
1186 * Remove the non-existent page (unless it's
1187 * in the top object) and move on down to the
1188 * next object (if there is one).
1189 */
1190 #if TRACEFAULTPAGE
1191 dbgTrace(0xBEEF0008, (unsigned int) m, (unsigned int) object->shadow); /* (TEST/DEBUG) */
1192 #endif
1193 next_object = object->shadow;
1194
1195 if (next_object == VM_OBJECT_NULL) {
1196 /*
1197 * Absent page at bottom of shadow
1198 * chain; zero fill the page we left
1199 * busy in the first object, and free
1200 * the absent page.
1201 */
1202 assert(!must_be_resident);
1203
1204 /*
1205 * check for any conditions that prevent
1206 * us from creating a new zero-fill page
1207 * vm_fault_check will do all of the
1208 * fault cleanup in the case of an error condition
1209 * including resetting the thread_interrupt_level
1210 */
1211 error = vm_fault_check(object, m, first_m, interruptible_state, (type_of_fault == NULL) ? TRUE : FALSE);
1212
1213 if (error != VM_FAULT_SUCCESS)
1214 return (error);
1215
1216 XPR(XPR_VM_FAULT,
1217 "vm_f_page: zero obj 0x%X, off 0x%X, page 0x%X, first_obj 0x%X\n",
1218 object, offset,
1219 m,
1220 first_object, 0);
1221
1222 if (object != first_object) {
1223 /*
1224 * free the absent page we just found
1225 */
1226 VM_PAGE_FREE(m);
1227
1228 /*
1229 * drop reference and lock on current object
1230 */
1231 vm_object_paging_end(object);
1232 vm_object_unlock(object);
1233
1234 /*
1235 * grab the original page we
1236 * 'soldered' in place and
1237 * retake lock on 'first_object'
1238 */
1239 m = first_m;
1240 first_m = VM_PAGE_NULL;
1241
1242 object = first_object;
1243 offset = first_offset;
1244
1245 vm_object_lock(object);
1246 } else {
1247 /*
1248 * we're going to use the absent page we just found
1249 * so convert it to a 'busy' page
1250 */
1251 m->absent = FALSE;
1252 m->busy = TRUE;
1253 }
1254 if (fault_info->mark_zf_absent && no_zero_fill == TRUE)
1255 m->absent = TRUE;
1256 /*
1257 * zero-fill the page and put it on
1258 * the correct paging queue
1259 */
1260 my_fault = vm_fault_zero_page(m, no_zero_fill);
1261
1262 break;
1263 } else {
1264 if (must_be_resident)
1265 vm_object_paging_end(object);
1266 else if (object != first_object) {
1267 vm_object_paging_end(object);
1268 VM_PAGE_FREE(m);
1269 } else {
1270 first_m = m;
1271 m->absent = FALSE;
1272 m->busy = TRUE;
1273
1274 vm_page_lockspin_queues();
1275 vm_page_queues_remove(m, FALSE);
1276 vm_page_unlock_queues();
1277 }
1278 XPR(XPR_VM_FAULT,
1279 "vm_f_page: unavail obj 0x%X, off 0x%X, next_obj 0x%X, newoff 0x%X\n",
1280 object, offset,
1281 next_object,
1282 offset+object->vo_shadow_offset,0);
1283
1284 offset += object->vo_shadow_offset;
1285 fault_info->lo_offset += object->vo_shadow_offset;
1286 fault_info->hi_offset += object->vo_shadow_offset;
1287 access_required = VM_PROT_READ;
1288
1289 vm_object_lock(next_object);
1290 vm_object_unlock(object);
1291 object = next_object;
1292 vm_object_paging_begin(object);
1293
1294 /*
1295 * reset to default type of fault
1296 */
1297 my_fault = DBG_CACHE_HIT_FAULT;
1298
1299 continue;
1300 }
1301 }
1302 if ((m->cleaning)
1303 && ((object != first_object) || (object->copy != VM_OBJECT_NULL))
1304 && (fault_type & VM_PROT_WRITE)) {
1305 /*
1306 * This is a copy-on-write fault that will
1307 * cause us to revoke access to this page, but
1308 * this page is in the process of being cleaned
1309 * in a clustered pageout. We must wait until
1310 * the cleaning operation completes before
1311 * revoking access to the original page,
1312 * otherwise we might attempt to remove a
1313 * wired mapping.
1314 */
1315 #if TRACEFAULTPAGE
1316 dbgTrace(0xBEEF0009, (unsigned int) m, (unsigned int) offset); /* (TEST/DEBUG) */
1317 #endif
1318 XPR(XPR_VM_FAULT,
1319 "vm_f_page: cleaning obj 0x%X, offset 0x%X, page 0x%X\n",
1320 object, offset,
1321 m, 0, 0);
1322 /*
1323 * take an extra ref so that object won't die
1324 */
1325 vm_object_reference_locked(object);
1326
1327 vm_fault_cleanup(object, first_m);
1328
1329 counter(c_vm_fault_page_block_backoff_kernel++);
1330 vm_object_lock(object);
1331 assert(object->ref_count > 0);
1332
1333 m = vm_page_lookup(object, offset);
1334
1335 if (m != VM_PAGE_NULL && m->cleaning) {
1336 PAGE_ASSERT_WAIT(m, interruptible);
1337
1338 vm_object_unlock(object);
1339 wait_result = thread_block(THREAD_CONTINUE_NULL);
1340 vm_object_deallocate(object);
1341
1342 goto backoff;
1343 } else {
1344 vm_object_unlock(object);
1345
1346 vm_object_deallocate(object);
1347 thread_interrupt_level(interruptible_state);
1348
1349 return (VM_FAULT_RETRY);
1350 }
1351 }
1352 if (type_of_fault == NULL && (m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) &&
1353 !(fault_info != NULL && fault_info->stealth)) {
1354 /*
1355 * If we were passed a non-NULL pointer for
1356 * "type_of_fault", than we came from
1357 * vm_fault... we'll let it deal with
1358 * this condition, since it
1359 * needs to see m->speculative to correctly
1360 * account the pageins, otherwise...
1361 * take it off the speculative queue, we'll
1362 * let the caller of vm_fault_page deal
1363 * with getting it onto the correct queue
1364 *
1365 * If the caller specified in fault_info that
1366 * it wants a "stealth" fault, we also leave
1367 * the page in the speculative queue.
1368 */
1369 vm_page_lockspin_queues();
1370 if (m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q)
1371 vm_page_queues_remove(m, FALSE);
1372 vm_page_unlock_queues();
1373 }
1374 assert(object == VM_PAGE_OBJECT(m));
1375
1376 if (m->encrypted) {
1377 /*
1378 * ENCRYPTED SWAP:
1379 * the user needs access to a page that we
1380 * encrypted before paging it out.
1381 * Decrypt the page now.
1382 * Keep it busy to prevent anyone from
1383 * accessing it during the decryption.
1384 */
1385 m->busy = TRUE;
1386 vm_page_decrypt(m, 0);
1387 assert(m->busy);
1388 PAGE_WAKEUP_DONE(m);
1389
1390 /*
1391 * Retry from the top, in case
1392 * something changed while we were
1393 * decrypting.
1394 */
1395 continue;
1396 }
1397 ASSERT_PAGE_DECRYPTED(m);
1398
1399 if (object->code_signed) {
1400 /*
1401 * CODE SIGNING:
1402 * We just paged in a page from a signed
1403 * memory object but we don't need to
1404 * validate it now. We'll validate it if
1405 * when it gets mapped into a user address
1406 * space for the first time or when the page
1407 * gets copied to another object as a result
1408 * of a copy-on-write.
1409 */
1410 }
1411
1412 /*
1413 * We mark the page busy and leave it on
1414 * the pageout queues. If the pageout
1415 * deamon comes across it, then it will
1416 * remove the page from the queue, but not the object
1417 */
1418 #if TRACEFAULTPAGE
1419 dbgTrace(0xBEEF000B, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1420 #endif
1421 XPR(XPR_VM_FAULT,
1422 "vm_f_page: found page obj 0x%X, offset 0x%X, page 0x%X\n",
1423 object, offset, m, 0, 0);
1424 assert(!m->busy);
1425 assert(!m->absent);
1426
1427 m->busy = TRUE;
1428 break;
1429 }
1430
1431
1432 /*
1433 * we get here when there is no page present in the object at
1434 * the offset we're interested in... we'll allocate a page
1435 * at this point if the pager associated with
1436 * this object can provide the data or we're the top object...
1437 * object is locked; m == NULL
1438 */
1439 if (must_be_resident) {
1440 if (fault_type == VM_PROT_NONE &&
1441 object == kernel_object) {
1442 /*
1443 * We've been called from vm_fault_unwire()
1444 * while removing a map entry that was allocated
1445 * with KMA_KOBJECT and KMA_VAONLY. This page
1446 * is not present and there's nothing more to
1447 * do here (nothing to unwire).
1448 */
1449 vm_fault_cleanup(object, first_m);
1450 thread_interrupt_level(interruptible_state);
1451
1452 return VM_FAULT_MEMORY_ERROR;
1453 }
1454
1455 goto dont_look_for_page;
1456 }
1457 data_supply = FALSE;
1458
1459 look_for_page = (object->pager_created && (MUST_ASK_PAGER(object, offset, external_state) == TRUE) && !data_supply);
1460
1461 #if TRACEFAULTPAGE
1462 dbgTrace(0xBEEF000C, (unsigned int) look_for_page, (unsigned int) object); /* (TEST/DEBUG) */
1463 #endif
1464 if (!look_for_page && object == first_object && !object->phys_contiguous) {
1465 /*
1466 * Allocate a new page for this object/offset pair as a placeholder
1467 */
1468 m = vm_page_grab_options(grab_options);
1469 #if TRACEFAULTPAGE
1470 dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
1471 #endif
1472 if (m == VM_PAGE_NULL) {
1473
1474 vm_fault_cleanup(object, first_m);
1475 thread_interrupt_level(interruptible_state);
1476
1477 return (VM_FAULT_MEMORY_SHORTAGE);
1478 }
1479
1480 if (fault_info && fault_info->batch_pmap_op == TRUE) {
1481 vm_page_insert_internal(m, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, TRUE, FALSE, NULL);
1482 } else {
1483 vm_page_insert(m, object, offset);
1484 }
1485 }
1486 if (look_for_page) {
1487 kern_return_t rc;
1488 int my_fault_type;
1489
1490 /*
1491 * If the memory manager is not ready, we
1492 * cannot make requests.
1493 */
1494 if (!object->pager_ready) {
1495 #if TRACEFAULTPAGE
1496 dbgTrace(0xBEEF000E, (unsigned int) 0, (unsigned int) 0); /* (TEST/DEBUG) */
1497 #endif
1498 if (m != VM_PAGE_NULL)
1499 VM_PAGE_FREE(m);
1500
1501 XPR(XPR_VM_FAULT,
1502 "vm_f_page: ready wait obj 0x%X, offset 0x%X\n",
1503 object, offset, 0, 0, 0);
1504
1505 /*
1506 * take an extra ref so object won't die
1507 */
1508 vm_object_reference_locked(object);
1509 vm_fault_cleanup(object, first_m);
1510 counter(c_vm_fault_page_block_backoff_kernel++);
1511
1512 vm_object_lock(object);
1513 assert(object->ref_count > 0);
1514
1515 if (!object->pager_ready) {
1516 wait_result = vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGER_READY, interruptible);
1517
1518 vm_object_unlock(object);
1519 if (wait_result == THREAD_WAITING)
1520 wait_result = thread_block(THREAD_CONTINUE_NULL);
1521 vm_object_deallocate(object);
1522
1523 goto backoff;
1524 } else {
1525 vm_object_unlock(object);
1526 vm_object_deallocate(object);
1527 thread_interrupt_level(interruptible_state);
1528
1529 return (VM_FAULT_RETRY);
1530 }
1531 }
1532 if (!object->internal && !object->phys_contiguous && object->paging_in_progress > vm_object_pagein_throttle) {
1533 /*
1534 * If there are too many outstanding page
1535 * requests pending on this external object, we
1536 * wait for them to be resolved now.
1537 */
1538 #if TRACEFAULTPAGE
1539 dbgTrace(0xBEEF0010, (unsigned int) m, (unsigned int) 0); /* (TEST/DEBUG) */
1540 #endif
1541 if (m != VM_PAGE_NULL)
1542 VM_PAGE_FREE(m);
1543 /*
1544 * take an extra ref so object won't die
1545 */
1546 vm_object_reference_locked(object);
1547
1548 vm_fault_cleanup(object, first_m);
1549
1550 counter(c_vm_fault_page_block_backoff_kernel++);
1551
1552 vm_object_lock(object);
1553 assert(object->ref_count > 0);
1554
1555 if (object->paging_in_progress >= vm_object_pagein_throttle) {
1556 vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGING_ONLY_IN_PROGRESS, interruptible);
1557
1558 vm_object_unlock(object);
1559 wait_result = thread_block(THREAD_CONTINUE_NULL);
1560 vm_object_deallocate(object);
1561
1562 goto backoff;
1563 } else {
1564 vm_object_unlock(object);
1565 vm_object_deallocate(object);
1566 thread_interrupt_level(interruptible_state);
1567
1568 return (VM_FAULT_RETRY);
1569 }
1570 }
1571 if (object->internal) {
1572 int compressed_count_delta;
1573
1574 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
1575
1576 if (m == VM_PAGE_NULL) {
1577 /*
1578 * Allocate a new page for this object/offset pair as a placeholder
1579 */
1580 m = vm_page_grab_options(grab_options);
1581 #if TRACEFAULTPAGE
1582 dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object); /* (TEST/DEBUG) */
1583 #endif
1584 if (m == VM_PAGE_NULL) {
1585
1586 vm_fault_cleanup(object, first_m);
1587 thread_interrupt_level(interruptible_state);
1588
1589 return (VM_FAULT_MEMORY_SHORTAGE);
1590 }
1591
1592 m->absent = TRUE;
1593 if (fault_info && fault_info->batch_pmap_op == TRUE) {
1594 vm_page_insert_internal(m, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, TRUE, FALSE, NULL);
1595 } else {
1596 vm_page_insert(m, object, offset);
1597 }
1598 }
1599 assert(m->busy);
1600
1601 m->absent = TRUE;
1602 pager = object->pager;
1603
1604 assert(object->paging_in_progress > 0);
1605 vm_object_unlock(object);
1606
1607 rc = vm_compressor_pager_get(
1608 pager,
1609 offset + object->paging_offset,
1610 VM_PAGE_GET_PHYS_PAGE(m),
1611 &my_fault_type,
1612 0,
1613 &compressed_count_delta);
1614
1615 if (type_of_fault == NULL) {
1616 int throttle_delay;
1617
1618 /*
1619 * we weren't called from vm_fault, so we
1620 * need to apply page creation throttling
1621 * do it before we re-acquire any locks
1622 */
1623 if (my_fault_type == DBG_COMPRESSOR_FAULT) {
1624 if ((throttle_delay = vm_page_throttled(TRUE))) {
1625 VM_DEBUG_EVENT(vmf_compressordelay, VMF_COMPRESSORDELAY, DBG_FUNC_NONE, throttle_delay, 0, 1, 0);
1626 delay(throttle_delay);
1627 }
1628 }
1629 }
1630 vm_object_lock(object);
1631 assert(object->paging_in_progress > 0);
1632
1633 vm_compressor_pager_count(
1634 pager,
1635 compressed_count_delta,
1636 FALSE, /* shared_lock */
1637 object);
1638
1639 switch (rc) {
1640 case KERN_SUCCESS:
1641 m->absent = FALSE;
1642 m->dirty = TRUE;
1643 if ((object->wimg_bits &
1644 VM_WIMG_MASK) !=
1645 VM_WIMG_USE_DEFAULT) {
1646 /*
1647 * If the page is not cacheable,
1648 * we can't let its contents
1649 * linger in the data cache
1650 * after the decompression.
1651 */
1652 pmap_sync_page_attributes_phys(
1653 VM_PAGE_GET_PHYS_PAGE(m));
1654 } else {
1655 m->written_by_kernel = TRUE;
1656 }
1657
1658 /*
1659 * If the object is purgeable, its
1660 * owner's purgeable ledgers have been
1661 * updated in vm_page_insert() but the
1662 * page was also accounted for in a
1663 * "compressed purgeable" ledger, so
1664 * update that now.
1665 */
1666 if ((object->purgable !=
1667 VM_PURGABLE_DENY) &&
1668 (object->vo_purgeable_owner !=
1669 NULL)) {
1670 /*
1671 * One less compressed
1672 * purgeable page.
1673 */
1674 vm_purgeable_compressed_update(
1675 object,
1676 -1);
1677 }
1678
1679 break;
1680 case KERN_MEMORY_FAILURE:
1681 m->unusual = TRUE;
1682 m->error = TRUE;
1683 m->absent = FALSE;
1684 break;
1685 case KERN_MEMORY_ERROR:
1686 assert(m->absent);
1687 break;
1688 default:
1689 panic("vm_fault_page(): unexpected "
1690 "error %d from "
1691 "vm_compressor_pager_get()\n",
1692 rc);
1693 }
1694 PAGE_WAKEUP_DONE(m);
1695
1696 rc = KERN_SUCCESS;
1697 goto data_requested;
1698 }
1699 my_fault_type = DBG_PAGEIN_FAULT;
1700
1701 if (m != VM_PAGE_NULL) {
1702 VM_PAGE_FREE(m);
1703 m = VM_PAGE_NULL;
1704 }
1705
1706 #if TRACEFAULTPAGE
1707 dbgTrace(0xBEEF0012, (unsigned int) object, (unsigned int) 0); /* (TEST/DEBUG) */
1708 #endif
1709
1710 /*
1711 * It's possible someone called vm_object_destroy while we weren't
1712 * holding the object lock. If that has happened, then bail out
1713 * here.
1714 */
1715
1716 pager = object->pager;
1717
1718 if (pager == MEMORY_OBJECT_NULL) {
1719 vm_fault_cleanup(object, first_m);
1720 thread_interrupt_level(interruptible_state);
1721 return VM_FAULT_MEMORY_ERROR;
1722 }
1723
1724 /*
1725 * We have an absent page in place for the faulting offset,
1726 * so we can release the object lock.
1727 */
1728
1729 vm_object_unlock(object);
1730
1731 /*
1732 * If this object uses a copy_call strategy,
1733 * and we are interested in a copy of this object
1734 * (having gotten here only by following a
1735 * shadow chain), then tell the memory manager
1736 * via a flag added to the desired_access
1737 * parameter, so that it can detect a race
1738 * between our walking down the shadow chain
1739 * and its pushing pages up into a copy of
1740 * the object that it manages.
1741 */
1742 if (object->copy_strategy == MEMORY_OBJECT_COPY_CALL && object != first_object)
1743 wants_copy_flag = VM_PROT_WANTS_COPY;
1744 else
1745 wants_copy_flag = VM_PROT_NONE;
1746
1747 XPR(XPR_VM_FAULT,
1748 "vm_f_page: data_req obj 0x%X, offset 0x%X, page 0x%X, acc %d\n",
1749 object, offset, m,
1750 access_required | wants_copy_flag, 0);
1751
1752 if (object->copy == first_object) {
1753 /*
1754 * if we issue the memory_object_data_request in
1755 * this state, we are subject to a deadlock with
1756 * the underlying filesystem if it is trying to
1757 * shrink the file resulting in a push of pages
1758 * into the copy object... that push will stall
1759 * on the placeholder page, and if the pushing thread
1760 * is holding a lock that is required on the pagein
1761 * path (such as a truncate lock), we'll deadlock...
1762 * to avoid this potential deadlock, we throw away
1763 * our placeholder page before calling memory_object_data_request
1764 * and force this thread to retry the vm_fault_page after
1765 * we have issued the I/O. the second time through this path
1766 * we will find the page already in the cache (presumably still
1767 * busy waiting for the I/O to complete) and then complete
1768 * the fault w/o having to go through memory_object_data_request again
1769 */
1770 assert(first_m != VM_PAGE_NULL);
1771 assert(VM_PAGE_OBJECT(first_m) == first_object);
1772
1773 vm_object_lock(first_object);
1774 VM_PAGE_FREE(first_m);
1775 vm_object_paging_end(first_object);
1776 vm_object_unlock(first_object);
1777
1778 first_m = VM_PAGE_NULL;
1779 force_fault_retry = TRUE;
1780
1781 vm_fault_page_forced_retry++;
1782 }
1783
1784 if (data_already_requested == TRUE) {
1785 orig_behavior = fault_info->behavior;
1786 orig_cluster_size = fault_info->cluster_size;
1787
1788 fault_info->behavior = VM_BEHAVIOR_RANDOM;
1789 fault_info->cluster_size = PAGE_SIZE;
1790 }
1791 /*
1792 * Call the memory manager to retrieve the data.
1793 */
1794 rc = memory_object_data_request(
1795 pager,
1796 offset + object->paging_offset,
1797 PAGE_SIZE,
1798 access_required | wants_copy_flag,
1799 (memory_object_fault_info_t)fault_info);
1800
1801 if (data_already_requested == TRUE) {
1802 fault_info->behavior = orig_behavior;
1803 fault_info->cluster_size = orig_cluster_size;
1804 } else
1805 data_already_requested = TRUE;
1806
1807 DTRACE_VM2(maj_fault, int, 1, (uint64_t *), NULL);
1808 #if TRACEFAULTPAGE
1809 dbgTrace(0xBEEF0013, (unsigned int) object, (unsigned int) rc); /* (TEST/DEBUG) */
1810 #endif
1811 vm_object_lock(object);
1812
1813 data_requested:
1814 if (rc != KERN_SUCCESS) {
1815
1816 vm_fault_cleanup(object, first_m);
1817 thread_interrupt_level(interruptible_state);
1818
1819 return ((rc == MACH_SEND_INTERRUPTED) ?
1820 VM_FAULT_INTERRUPTED :
1821 VM_FAULT_MEMORY_ERROR);
1822 } else {
1823 clock_sec_t tv_sec;
1824 clock_usec_t tv_usec;
1825
1826 if (my_fault_type == DBG_PAGEIN_FAULT) {
1827 clock_get_system_microtime(&tv_sec, &tv_usec);
1828 current_thread()->t_page_creation_time = tv_sec;
1829 current_thread()->t_page_creation_count = 0;
1830 }
1831 }
1832 if ((interruptible != THREAD_UNINT) && (current_thread()->sched_flags & TH_SFLAG_ABORT)) {
1833
1834 vm_fault_cleanup(object, first_m);
1835 thread_interrupt_level(interruptible_state);
1836
1837 return (VM_FAULT_INTERRUPTED);
1838 }
1839 if (force_fault_retry == TRUE) {
1840
1841 vm_fault_cleanup(object, first_m);
1842 thread_interrupt_level(interruptible_state);
1843
1844 return (VM_FAULT_RETRY);
1845 }
1846 if (m == VM_PAGE_NULL && object->phys_contiguous) {
1847 /*
1848 * No page here means that the object we
1849 * initially looked up was "physically
1850 * contiguous" (i.e. device memory). However,
1851 * with Virtual VRAM, the object might not
1852 * be backed by that device memory anymore,
1853 * so we're done here only if the object is
1854 * still "phys_contiguous".
1855 * Otherwise, if the object is no longer
1856 * "phys_contiguous", we need to retry the
1857 * page fault against the object's new backing
1858 * store (different memory object).
1859 */
1860 phys_contig_object:
1861 goto done;
1862 }
1863 /*
1864 * potentially a pagein fault
1865 * if we make it through the state checks
1866 * above, than we'll count it as such
1867 */
1868 my_fault = my_fault_type;
1869
1870 /*
1871 * Retry with same object/offset, since new data may
1872 * be in a different page (i.e., m is meaningless at
1873 * this point).
1874 */
1875 continue;
1876 }
1877 dont_look_for_page:
1878 /*
1879 * We get here if the object has no pager, or an existence map
1880 * exists and indicates the page isn't present on the pager
1881 * or we're unwiring a page. If a pager exists, but there
1882 * is no existence map, then the m->absent case above handles
1883 * the ZF case when the pager can't provide the page
1884 */
1885 #if TRACEFAULTPAGE
1886 dbgTrace(0xBEEF0014, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
1887 #endif
1888 if (object == first_object)
1889 first_m = m;
1890 else
1891 assert(m == VM_PAGE_NULL);
1892
1893 XPR(XPR_VM_FAULT,
1894 "vm_f_page: no pager obj 0x%X, offset 0x%X, page 0x%X, next_obj 0x%X\n",
1895 object, offset, m,
1896 object->shadow, 0);
1897
1898 next_object = object->shadow;
1899
1900 if (next_object == VM_OBJECT_NULL) {
1901 /*
1902 * we've hit the bottom of the shadown chain,
1903 * fill the page in the top object with zeros.
1904 */
1905 assert(!must_be_resident);
1906
1907 if (object != first_object) {
1908 vm_object_paging_end(object);
1909 vm_object_unlock(object);
1910
1911 object = first_object;
1912 offset = first_offset;
1913 vm_object_lock(object);
1914 }
1915 m = first_m;
1916 assert(VM_PAGE_OBJECT(m) == object);
1917 first_m = VM_PAGE_NULL;
1918
1919 /*
1920 * check for any conditions that prevent
1921 * us from creating a new zero-fill page
1922 * vm_fault_check will do all of the
1923 * fault cleanup in the case of an error condition
1924 * including resetting the thread_interrupt_level
1925 */
1926 error = vm_fault_check(object, m, first_m, interruptible_state, (type_of_fault == NULL) ? TRUE : FALSE);
1927
1928 if (error != VM_FAULT_SUCCESS)
1929 return (error);
1930
1931 if (m == VM_PAGE_NULL) {
1932 m = vm_page_grab_options(grab_options);
1933
1934 if (m == VM_PAGE_NULL) {
1935 vm_fault_cleanup(object, VM_PAGE_NULL);
1936 thread_interrupt_level(interruptible_state);
1937
1938 return (VM_FAULT_MEMORY_SHORTAGE);
1939 }
1940 vm_page_insert(m, object, offset);
1941 }
1942 if (fault_info->mark_zf_absent && no_zero_fill == TRUE)
1943 m->absent = TRUE;
1944
1945 my_fault = vm_fault_zero_page(m, no_zero_fill);
1946
1947 break;
1948
1949 } else {
1950 /*
1951 * Move on to the next object. Lock the next
1952 * object before unlocking the current one.
1953 */
1954 if ((object != first_object) || must_be_resident)
1955 vm_object_paging_end(object);
1956
1957 offset += object->vo_shadow_offset;
1958 fault_info->lo_offset += object->vo_shadow_offset;
1959 fault_info->hi_offset += object->vo_shadow_offset;
1960 access_required = VM_PROT_READ;
1961
1962 vm_object_lock(next_object);
1963 vm_object_unlock(object);
1964
1965 object = next_object;
1966 vm_object_paging_begin(object);
1967 }
1968 }
1969
1970 /*
1971 * PAGE HAS BEEN FOUND.
1972 *
1973 * This page (m) is:
1974 * busy, so that we can play with it;
1975 * not absent, so that nobody else will fill it;
1976 * possibly eligible for pageout;
1977 *
1978 * The top-level page (first_m) is:
1979 * VM_PAGE_NULL if the page was found in the
1980 * top-level object;
1981 * busy, not absent, and ineligible for pageout.
1982 *
1983 * The current object (object) is locked. A paging
1984 * reference is held for the current and top-level
1985 * objects.
1986 */
1987
1988 #if TRACEFAULTPAGE
1989 dbgTrace(0xBEEF0015, (unsigned int) object, (unsigned int) m); /* (TEST/DEBUG) */
1990 #endif
1991 #if EXTRA_ASSERTIONS
1992 assert(m->busy && !m->absent);
1993 assert((first_m == VM_PAGE_NULL) ||
1994 (first_m->busy && !first_m->absent &&
1995 !first_m->active && !first_m->inactive && !first_m->secluded));
1996 #endif /* EXTRA_ASSERTIONS */
1997
1998 /*
1999 * ENCRYPTED SWAP:
2000 * If we found a page, we must have decrypted it before we
2001 * get here...
2002 */
2003 ASSERT_PAGE_DECRYPTED(m);
2004
2005 XPR(XPR_VM_FAULT,
2006 "vm_f_page: FOUND obj 0x%X, off 0x%X, page 0x%X, 1_obj 0x%X, 1_m 0x%X\n",
2007 object, offset, m,
2008 first_object, first_m);
2009
2010 /*
2011 * If the page is being written, but isn't
2012 * already owned by the top-level object,
2013 * we have to copy it into a new page owned
2014 * by the top-level object.
2015 */
2016 if (object != first_object) {
2017
2018 #if TRACEFAULTPAGE
2019 dbgTrace(0xBEEF0016, (unsigned int) object, (unsigned int) fault_type); /* (TEST/DEBUG) */
2020 #endif
2021 if (fault_type & VM_PROT_WRITE) {
2022 vm_page_t copy_m;
2023
2024 /*
2025 * We only really need to copy if we
2026 * want to write it.
2027 */
2028 assert(!must_be_resident);
2029
2030 /*
2031 * are we protecting the system from
2032 * backing store exhaustion. If so
2033 * sleep unless we are privileged.
2034 */
2035 if (vm_backing_store_low) {
2036 if (!(current_task()->priv_flags & VM_BACKING_STORE_PRIV)) {
2037
2038 RELEASE_PAGE(m);
2039 vm_fault_cleanup(object, first_m);
2040
2041 assert_wait((event_t)&vm_backing_store_low, THREAD_UNINT);
2042
2043 thread_block(THREAD_CONTINUE_NULL);
2044 thread_interrupt_level(interruptible_state);
2045
2046 return (VM_FAULT_RETRY);
2047 }
2048 }
2049 /*
2050 * If we try to collapse first_object at this
2051 * point, we may deadlock when we try to get
2052 * the lock on an intermediate object (since we
2053 * have the bottom object locked). We can't
2054 * unlock the bottom object, because the page
2055 * we found may move (by collapse) if we do.
2056 *
2057 * Instead, we first copy the page. Then, when
2058 * we have no more use for the bottom object,
2059 * we unlock it and try to collapse.
2060 *
2061 * Note that we copy the page even if we didn't
2062 * need to... that's the breaks.
2063 */
2064
2065 /*
2066 * Allocate a page for the copy
2067 */
2068 copy_m = vm_page_grab_options(grab_options);
2069
2070 if (copy_m == VM_PAGE_NULL) {
2071 RELEASE_PAGE(m);
2072
2073 vm_fault_cleanup(object, first_m);
2074 thread_interrupt_level(interruptible_state);
2075
2076 return (VM_FAULT_MEMORY_SHORTAGE);
2077 }
2078 XPR(XPR_VM_FAULT,
2079 "vm_f_page: page_copy obj 0x%X, offset 0x%X, m 0x%X, copy_m 0x%X\n",
2080 object, offset,
2081 m, copy_m, 0);
2082
2083 vm_page_copy(m, copy_m);
2084
2085 /*
2086 * If another map is truly sharing this
2087 * page with us, we have to flush all
2088 * uses of the original page, since we
2089 * can't distinguish those which want the
2090 * original from those which need the
2091 * new copy.
2092 *
2093 * XXXO If we know that only one map has
2094 * access to this page, then we could
2095 * avoid the pmap_disconnect() call.
2096 */
2097 if (m->pmapped)
2098 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
2099
2100 if (m->clustered) {
2101 VM_PAGE_COUNT_AS_PAGEIN(m);
2102 VM_PAGE_CONSUME_CLUSTERED(m);
2103 }
2104 assert(!m->cleaning);
2105
2106 /*
2107 * We no longer need the old page or object.
2108 */
2109 RELEASE_PAGE(m);
2110
2111 /*
2112 * This check helps with marking the object as having a sequential pattern
2113 * Normally we'll miss doing this below because this fault is about COW to
2114 * the first_object i.e. bring page in from disk, push to object above but
2115 * don't update the file object's sequential pattern.
2116 */
2117 if (object->internal == FALSE) {
2118 vm_fault_is_sequential(object, offset, fault_info->behavior);
2119 }
2120
2121 vm_object_paging_end(object);
2122 vm_object_unlock(object);
2123
2124 my_fault = DBG_COW_FAULT;
2125 VM_STAT_INCR(cow_faults);
2126 DTRACE_VM2(cow_fault, int, 1, (uint64_t *), NULL);
2127 current_task()->cow_faults++;
2128
2129 object = first_object;
2130 offset = first_offset;
2131
2132 vm_object_lock(object);
2133 /*
2134 * get rid of the place holder
2135 * page that we soldered in earlier
2136 */
2137 VM_PAGE_FREE(first_m);
2138 first_m = VM_PAGE_NULL;
2139
2140 /*
2141 * and replace it with the
2142 * page we just copied into
2143 */
2144 assert(copy_m->busy);
2145 vm_page_insert(copy_m, object, offset);
2146 SET_PAGE_DIRTY(copy_m, TRUE);
2147
2148 m = copy_m;
2149 /*
2150 * Now that we've gotten the copy out of the
2151 * way, let's try to collapse the top object.
2152 * But we have to play ugly games with
2153 * paging_in_progress to do that...
2154 */
2155 vm_object_paging_end(object);
2156 vm_object_collapse(object, offset, TRUE);
2157 vm_object_paging_begin(object);
2158
2159 } else
2160 *protection &= (~VM_PROT_WRITE);
2161 }
2162 /*
2163 * Now check whether the page needs to be pushed into the
2164 * copy object. The use of asymmetric copy on write for
2165 * shared temporary objects means that we may do two copies to
2166 * satisfy the fault; one above to get the page from a
2167 * shadowed object, and one here to push it into the copy.
2168 */
2169 try_failed_count = 0;
2170
2171 while ((copy_object = first_object->copy) != VM_OBJECT_NULL) {
2172 vm_object_offset_t copy_offset;
2173 vm_page_t copy_m;
2174
2175 #if TRACEFAULTPAGE
2176 dbgTrace(0xBEEF0017, (unsigned int) copy_object, (unsigned int) fault_type); /* (TEST/DEBUG) */
2177 #endif
2178 /*
2179 * If the page is being written, but hasn't been
2180 * copied to the copy-object, we have to copy it there.
2181 */
2182 if ((fault_type & VM_PROT_WRITE) == 0) {
2183 *protection &= ~VM_PROT_WRITE;
2184 break;
2185 }
2186
2187 /*
2188 * If the page was guaranteed to be resident,
2189 * we must have already performed the copy.
2190 */
2191 if (must_be_resident)
2192 break;
2193
2194 /*
2195 * Try to get the lock on the copy_object.
2196 */
2197 if (!vm_object_lock_try(copy_object)) {
2198
2199 vm_object_unlock(object);
2200 try_failed_count++;
2201
2202 mutex_pause(try_failed_count); /* wait a bit */
2203 vm_object_lock(object);
2204
2205 continue;
2206 }
2207 try_failed_count = 0;
2208
2209 /*
2210 * Make another reference to the copy-object,
2211 * to keep it from disappearing during the
2212 * copy.
2213 */
2214 vm_object_reference_locked(copy_object);
2215
2216 /*
2217 * Does the page exist in the copy?
2218 */
2219 copy_offset = first_offset - copy_object->vo_shadow_offset;
2220
2221 if (copy_object->vo_size <= copy_offset)
2222 /*
2223 * Copy object doesn't cover this page -- do nothing.
2224 */
2225 ;
2226 else if ((copy_m = vm_page_lookup(copy_object, copy_offset)) != VM_PAGE_NULL) {
2227 /*
2228 * Page currently exists in the copy object
2229 */
2230 if (copy_m->busy) {
2231 /*
2232 * If the page is being brought
2233 * in, wait for it and then retry.
2234 */
2235 RELEASE_PAGE(m);
2236
2237 /*
2238 * take an extra ref so object won't die
2239 */
2240 vm_object_reference_locked(copy_object);
2241 vm_object_unlock(copy_object);
2242 vm_fault_cleanup(object, first_m);
2243 counter(c_vm_fault_page_block_backoff_kernel++);
2244
2245 vm_object_lock(copy_object);
2246 assert(copy_object->ref_count > 0);
2247 VM_OBJ_RES_DECR(copy_object);
2248 vm_object_lock_assert_exclusive(copy_object);
2249 copy_object->ref_count--;
2250 assert(copy_object->ref_count > 0);
2251 copy_m = vm_page_lookup(copy_object, copy_offset);
2252 /*
2253 * ENCRYPTED SWAP:
2254 * it's OK if the "copy_m" page is encrypted,
2255 * because we're not moving it nor handling its
2256 * contents.
2257 */
2258 if (copy_m != VM_PAGE_NULL && copy_m->busy) {
2259 PAGE_ASSERT_WAIT(copy_m, interruptible);
2260
2261 vm_object_unlock(copy_object);
2262 wait_result = thread_block(THREAD_CONTINUE_NULL);
2263 vm_object_deallocate(copy_object);
2264
2265 goto backoff;
2266 } else {
2267 vm_object_unlock(copy_object);
2268 vm_object_deallocate(copy_object);
2269 thread_interrupt_level(interruptible_state);
2270
2271 return (VM_FAULT_RETRY);
2272 }
2273 }
2274 }
2275 else if (!PAGED_OUT(copy_object, copy_offset)) {
2276 /*
2277 * If PAGED_OUT is TRUE, then the page used to exist
2278 * in the copy-object, and has already been paged out.
2279 * We don't need to repeat this. If PAGED_OUT is
2280 * FALSE, then either we don't know (!pager_created,
2281 * for example) or it hasn't been paged out.
2282 * (VM_EXTERNAL_STATE_UNKNOWN||VM_EXTERNAL_STATE_ABSENT)
2283 * We must copy the page to the copy object.
2284 */
2285
2286 if (vm_backing_store_low) {
2287 /*
2288 * we are protecting the system from
2289 * backing store exhaustion. If so
2290 * sleep unless we are privileged.
2291 */
2292 if (!(current_task()->priv_flags & VM_BACKING_STORE_PRIV)) {
2293 assert_wait((event_t)&vm_backing_store_low, THREAD_UNINT);
2294
2295 RELEASE_PAGE(m);
2296 VM_OBJ_RES_DECR(copy_object);
2297 vm_object_lock_assert_exclusive(copy_object);
2298 copy_object->ref_count--;
2299 assert(copy_object->ref_count > 0);
2300
2301 vm_object_unlock(copy_object);
2302 vm_fault_cleanup(object, first_m);
2303 thread_block(THREAD_CONTINUE_NULL);
2304 thread_interrupt_level(interruptible_state);
2305
2306 return (VM_FAULT_RETRY);
2307 }
2308 }
2309 /*
2310 * Allocate a page for the copy
2311 */
2312 copy_m = vm_page_alloc(copy_object, copy_offset);
2313
2314 if (copy_m == VM_PAGE_NULL) {
2315 RELEASE_PAGE(m);
2316
2317 VM_OBJ_RES_DECR(copy_object);
2318 vm_object_lock_assert_exclusive(copy_object);
2319 copy_object->ref_count--;
2320 assert(copy_object->ref_count > 0);
2321
2322 vm_object_unlock(copy_object);
2323 vm_fault_cleanup(object, first_m);
2324 thread_interrupt_level(interruptible_state);
2325
2326 return (VM_FAULT_MEMORY_SHORTAGE);
2327 }
2328 /*
2329 * Must copy page into copy-object.
2330 */
2331 vm_page_copy(m, copy_m);
2332
2333 /*
2334 * If the old page was in use by any users
2335 * of the copy-object, it must be removed
2336 * from all pmaps. (We can't know which
2337 * pmaps use it.)
2338 */
2339 if (m->pmapped)
2340 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
2341
2342 if (m->clustered) {
2343 VM_PAGE_COUNT_AS_PAGEIN(m);
2344 VM_PAGE_CONSUME_CLUSTERED(m);
2345 }
2346 /*
2347 * If there's a pager, then immediately
2348 * page out this page, using the "initialize"
2349 * option. Else, we use the copy.
2350 */
2351 if ((!copy_object->pager_ready)
2352 || VM_COMPRESSOR_PAGER_STATE_GET(copy_object, copy_offset) == VM_EXTERNAL_STATE_ABSENT
2353 ) {
2354
2355 vm_page_lockspin_queues();
2356 assert(!m->cleaning);
2357 vm_page_activate(copy_m);
2358 vm_page_unlock_queues();
2359
2360 SET_PAGE_DIRTY(copy_m, TRUE);
2361 PAGE_WAKEUP_DONE(copy_m);
2362
2363 } else {
2364
2365 assert(copy_m->busy == TRUE);
2366 assert(!m->cleaning);
2367
2368 /*
2369 * dirty is protected by the object lock
2370 */
2371 SET_PAGE_DIRTY(copy_m, TRUE);
2372
2373 /*
2374 * The page is already ready for pageout:
2375 * not on pageout queues and busy.
2376 * Unlock everything except the
2377 * copy_object itself.
2378 */
2379 vm_object_unlock(object);
2380
2381 /*
2382 * Write the page to the copy-object,
2383 * flushing it from the kernel.
2384 */
2385 vm_pageout_initialize_page(copy_m);
2386
2387 /*
2388 * Since the pageout may have
2389 * temporarily dropped the
2390 * copy_object's lock, we
2391 * check whether we'll have
2392 * to deallocate the hard way.
2393 */
2394 if ((copy_object->shadow != object) || (copy_object->ref_count == 1)) {
2395 vm_object_unlock(copy_object);
2396 vm_object_deallocate(copy_object);
2397 vm_object_lock(object);
2398
2399 continue;
2400 }
2401 /*
2402 * Pick back up the old object's
2403 * lock. [It is safe to do so,
2404 * since it must be deeper in the
2405 * object tree.]
2406 */
2407 vm_object_lock(object);
2408 }
2409
2410 /*
2411 * Because we're pushing a page upward
2412 * in the object tree, we must restart
2413 * any faults that are waiting here.
2414 * [Note that this is an expansion of
2415 * PAGE_WAKEUP that uses the THREAD_RESTART
2416 * wait result]. Can't turn off the page's
2417 * busy bit because we're not done with it.
2418 */
2419 if (m->wanted) {
2420 m->wanted = FALSE;
2421 thread_wakeup_with_result((event_t) m, THREAD_RESTART);
2422 }
2423 }
2424 /*
2425 * The reference count on copy_object must be
2426 * at least 2: one for our extra reference,
2427 * and at least one from the outside world
2428 * (we checked that when we last locked
2429 * copy_object).
2430 */
2431 vm_object_lock_assert_exclusive(copy_object);
2432 copy_object->ref_count--;
2433 assert(copy_object->ref_count > 0);
2434
2435 VM_OBJ_RES_DECR(copy_object);
2436 vm_object_unlock(copy_object);
2437
2438 break;
2439 }
2440
2441 done:
2442 *result_page = m;
2443 *top_page = first_m;
2444
2445 XPR(XPR_VM_FAULT,
2446 "vm_f_page: DONE obj 0x%X, offset 0x%X, m 0x%X, first_m 0x%X\n",
2447 object, offset, m, first_m, 0);
2448
2449 if (m != VM_PAGE_NULL) {
2450 assert(VM_PAGE_OBJECT(m) == object);
2451
2452 retval = VM_FAULT_SUCCESS;
2453
2454 if (my_fault == DBG_PAGEIN_FAULT) {
2455
2456 VM_PAGE_COUNT_AS_PAGEIN(m);
2457
2458 if (object->internal)
2459 my_fault = DBG_PAGEIND_FAULT;
2460 else
2461 my_fault = DBG_PAGEINV_FAULT;
2462
2463 /*
2464 * evaluate access pattern and update state
2465 * vm_fault_deactivate_behind depends on the
2466 * state being up to date
2467 */
2468 vm_fault_is_sequential(object, offset, fault_info->behavior);
2469
2470 vm_fault_deactivate_behind(object, offset, fault_info->behavior);
2471 } else if (my_fault == DBG_COMPRESSOR_FAULT || my_fault == DBG_COMPRESSOR_SWAPIN_FAULT) {
2472
2473 VM_STAT_INCR(decompressions);
2474 }
2475 if (type_of_fault)
2476 *type_of_fault = my_fault;
2477 } else {
2478 retval = VM_FAULT_SUCCESS_NO_VM_PAGE;
2479 assert(first_m == VM_PAGE_NULL);
2480 assert(object == first_object);
2481 }
2482
2483 thread_interrupt_level(interruptible_state);
2484
2485 #if TRACEFAULTPAGE
2486 dbgTrace(0xBEEF001A, (unsigned int) VM_FAULT_SUCCESS, 0); /* (TEST/DEBUG) */
2487 #endif
2488 return retval;
2489
2490 backoff:
2491 thread_interrupt_level(interruptible_state);
2492
2493 if (wait_result == THREAD_INTERRUPTED)
2494 return (VM_FAULT_INTERRUPTED);
2495 return (VM_FAULT_RETRY);
2496
2497 #undef RELEASE_PAGE
2498 }
2499
2500
2501
2502 /*
2503 * CODE SIGNING:
2504 * When soft faulting a page, we have to validate the page if:
2505 * 1. the page is being mapped in user space
2506 * 2. the page hasn't already been found to be "tainted"
2507 * 3. the page belongs to a code-signed object
2508 * 4. the page has not been validated yet or has been mapped for write.
2509 */
2510 #define VM_FAULT_NEED_CS_VALIDATION(pmap, page, page_obj) \
2511 ((pmap) != kernel_pmap /*1*/ && \
2512 !(page)->cs_tainted /*2*/ && \
2513 (page_obj)->code_signed /*3*/ && \
2514 (!(page)->cs_validated || (page)->wpmapped /*4*/))
2515
2516
2517 /*
2518 * page queue lock must NOT be held
2519 * m->object must be locked
2520 *
2521 * NOTE: m->object could be locked "shared" only if we are called
2522 * from vm_fault() as part of a soft fault. If so, we must be
2523 * careful not to modify the VM object in any way that is not
2524 * legal under a shared lock...
2525 */
2526 extern int panic_on_cs_killed;
2527 extern int proc_selfpid(void);
2528 extern char *proc_name_address(void *p);
2529 unsigned long cs_enter_tainted_rejected = 0;
2530 unsigned long cs_enter_tainted_accepted = 0;
2531 kern_return_t
2532 vm_fault_enter(vm_page_t m,
2533 pmap_t pmap,
2534 vm_map_offset_t vaddr,
2535 vm_prot_t prot,
2536 vm_prot_t caller_prot,
2537 boolean_t wired,
2538 boolean_t change_wiring,
2539 boolean_t no_cache,
2540 boolean_t cs_bypass,
2541 __unused int user_tag,
2542 int pmap_options,
2543 boolean_t *need_retry,
2544 int *type_of_fault)
2545 {
2546 kern_return_t kr, pe_result;
2547 boolean_t previously_pmapped = m->pmapped;
2548 boolean_t must_disconnect = 0;
2549 boolean_t map_is_switched, map_is_switch_protected;
2550 int cs_enforcement_enabled;
2551 vm_prot_t fault_type;
2552 vm_object_t object;
2553
2554 fault_type = change_wiring ? VM_PROT_NONE : caller_prot;
2555 object = VM_PAGE_OBJECT(m);
2556
2557 vm_object_lock_assert_held(object);
2558 LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
2559
2560 if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
2561 assert(m->fictitious);
2562 return KERN_SUCCESS;
2563 }
2564
2565 if (*type_of_fault == DBG_ZERO_FILL_FAULT) {
2566
2567 vm_object_lock_assert_exclusive(object);
2568
2569 } else if ((fault_type & VM_PROT_WRITE) == 0 && !m->wpmapped) {
2570 /*
2571 * This is not a "write" fault, so we
2572 * might not have taken the object lock
2573 * exclusively and we might not be able
2574 * to update the "wpmapped" bit in
2575 * vm_fault_enter().
2576 * Let's just grant read access to
2577 * the page for now and we'll
2578 * soft-fault again if we need write
2579 * access later...
2580 */
2581 prot &= ~VM_PROT_WRITE;
2582 }
2583 if (m->pmapped == FALSE) {
2584
2585 if (m->clustered) {
2586 if (*type_of_fault == DBG_CACHE_HIT_FAULT) {
2587 /*
2588 * found it in the cache, but this
2589 * is the first fault-in of the page (m->pmapped == FALSE)
2590 * so it must have come in as part of
2591 * a cluster... account 1 pagein against it
2592 */
2593 if (object->internal)
2594 *type_of_fault = DBG_PAGEIND_FAULT;
2595 else
2596 *type_of_fault = DBG_PAGEINV_FAULT;
2597
2598 VM_PAGE_COUNT_AS_PAGEIN(m);
2599 }
2600 VM_PAGE_CONSUME_CLUSTERED(m);
2601 }
2602 }
2603
2604 if (*type_of_fault != DBG_COW_FAULT) {
2605 DTRACE_VM2(as_fault, int, 1, (uint64_t *), NULL);
2606
2607 if (pmap == kernel_pmap) {
2608 DTRACE_VM2(kernel_asflt, int, 1, (uint64_t *), NULL);
2609 }
2610 }
2611
2612 /* Validate code signature if necessary. */
2613 if (VM_FAULT_NEED_CS_VALIDATION(pmap, m, object)) {
2614 vm_object_lock_assert_exclusive(object);
2615
2616 if (m->cs_validated) {
2617 vm_cs_revalidates++;
2618 }
2619
2620 /* VM map is locked, so 1 ref will remain on VM object -
2621 * so no harm if vm_page_validate_cs drops the object lock */
2622 vm_page_validate_cs(m);
2623 }
2624
2625 #define page_immutable(m,prot) ((m)->cs_validated /*&& ((prot) & VM_PROT_EXECUTE)*/)
2626 #define page_nx(m) ((m)->cs_nx)
2627
2628 map_is_switched = ((pmap != vm_map_pmap(current_task()->map)) &&
2629 (pmap == vm_map_pmap(current_thread()->map)));
2630 map_is_switch_protected = current_thread()->map->switch_protect;
2631
2632 /* If the map is switched, and is switch-protected, we must protect
2633 * some pages from being write-faulted: immutable pages because by
2634 * definition they may not be written, and executable pages because that
2635 * would provide a way to inject unsigned code.
2636 * If the page is immutable, we can simply return. However, we can't
2637 * immediately determine whether a page is executable anywhere. But,
2638 * we can disconnect it everywhere and remove the executable protection
2639 * from the current map. We do that below right before we do the
2640 * PMAP_ENTER.
2641 */
2642 cs_enforcement_enabled = cs_enforcement(NULL);
2643
2644 if(cs_enforcement_enabled && map_is_switched &&
2645 map_is_switch_protected && page_immutable(m, prot) &&
2646 (prot & VM_PROT_WRITE))
2647 {
2648 return KERN_CODESIGN_ERROR;
2649 }
2650
2651 if (cs_enforcement_enabled && page_nx(m) && (prot & VM_PROT_EXECUTE)) {
2652 if (cs_debug)
2653 printf("page marked to be NX, not letting it be mapped EXEC\n");
2654 return KERN_CODESIGN_ERROR;
2655 }
2656
2657 if (cs_enforcement_enabled &&
2658 !m->cs_validated &&
2659 (prot & VM_PROT_EXECUTE) &&
2660 !(caller_prot & VM_PROT_EXECUTE)) {
2661 /*
2662 * FOURK PAGER:
2663 * This page has not been validated and will not be
2664 * allowed to be mapped for "execute".
2665 * But the caller did not request "execute" access for this
2666 * fault, so we should not raise a code-signing violation
2667 * (and possibly kill the process) below.
2668 * Instead, let's just remove the "execute" access request.
2669 *
2670 * This can happen on devices with a 4K page size if a 16K
2671 * page contains a mix of signed&executable and
2672 * unsigned&non-executable 4K pages, making the whole 16K
2673 * mapping "executable".
2674 */
2675 prot &= ~VM_PROT_EXECUTE;
2676 }
2677
2678 /* A page could be tainted, or pose a risk of being tainted later.
2679 * Check whether the receiving process wants it, and make it feel
2680 * the consequences (that hapens in cs_invalid_page()).
2681 * For CS Enforcement, two other conditions will
2682 * cause that page to be tainted as well:
2683 * - pmapping an unsigned page executable - this means unsigned code;
2684 * - writeable mapping of a validated page - the content of that page
2685 * can be changed without the kernel noticing, therefore unsigned
2686 * code can be created
2687 */
2688 if (!cs_bypass &&
2689 (m->cs_tainted ||
2690 (cs_enforcement_enabled &&
2691 (/* The page is unsigned and wants to be executable */
2692 (!m->cs_validated && (prot & VM_PROT_EXECUTE)) ||
2693 /* The page should be immutable, but is in danger of being modified
2694 * This is the case where we want policy from the code directory -
2695 * is the page immutable or not? For now we have to assume that
2696 * code pages will be immutable, data pages not.
2697 * We'll assume a page is a code page if it has a code directory
2698 * and we fault for execution.
2699 * That is good enough since if we faulted the code page for
2700 * writing in another map before, it is wpmapped; if we fault
2701 * it for writing in this map later it will also be faulted for executing
2702 * at the same time; and if we fault for writing in another map
2703 * later, we will disconnect it from this pmap so we'll notice
2704 * the change.
2705 */
2706 (page_immutable(m, prot) && ((prot & VM_PROT_WRITE) || m->wpmapped))
2707 ))
2708 ))
2709 {
2710 /* We will have a tainted page. Have to handle the special case
2711 * of a switched map now. If the map is not switched, standard
2712 * procedure applies - call cs_invalid_page().
2713 * If the map is switched, the real owner is invalid already.
2714 * There is no point in invalidating the switching process since
2715 * it will not be executing from the map. So we don't call
2716 * cs_invalid_page() in that case. */
2717 boolean_t reject_page, cs_killed;
2718 if(map_is_switched) {
2719 assert(pmap==vm_map_pmap(current_thread()->map));
2720 assert(!(prot & VM_PROT_WRITE) || (map_is_switch_protected == FALSE));
2721 reject_page = FALSE;
2722 } else {
2723 if (cs_debug > 5)
2724 printf("vm_fault: signed: %s validate: %s tainted: %s wpmapped: %s slid: %s prot: 0x%x\n",
2725 object->code_signed ? "yes" : "no",
2726 m->cs_validated ? "yes" : "no",
2727 m->cs_tainted ? "yes" : "no",
2728 m->wpmapped ? "yes" : "no",
2729 m->slid ? "yes" : "no",
2730 (int)prot);
2731 reject_page = cs_invalid_page((addr64_t) vaddr, &cs_killed);
2732 }
2733
2734 if (reject_page) {
2735 /* reject the invalid page: abort the page fault */
2736 int pid;
2737 const char *procname;
2738 task_t task;
2739 vm_object_t file_object, shadow;
2740 vm_object_offset_t file_offset;
2741 char *pathname, *filename;
2742 vm_size_t pathname_len, filename_len;
2743 boolean_t truncated_path;
2744 #define __PATH_MAX 1024
2745 struct timespec mtime, cs_mtime;
2746 int shadow_depth;
2747 os_reason_t codesigning_exit_reason = OS_REASON_NULL;
2748
2749 kr = KERN_CODESIGN_ERROR;
2750 cs_enter_tainted_rejected++;
2751
2752 /* get process name and pid */
2753 procname = "?";
2754 task = current_task();
2755 pid = proc_selfpid();
2756 if (task->bsd_info != NULL)
2757 procname = proc_name_address(task->bsd_info);
2758
2759 /* get file's VM object */
2760 file_object = object;
2761 file_offset = m->offset;
2762 for (shadow = file_object->shadow,
2763 shadow_depth = 0;
2764 shadow != VM_OBJECT_NULL;
2765 shadow = file_object->shadow,
2766 shadow_depth++) {
2767 vm_object_lock_shared(shadow);
2768 if (file_object != object) {
2769 vm_object_unlock(file_object);
2770 }
2771 file_offset += file_object->vo_shadow_offset;
2772 file_object = shadow;
2773 }
2774
2775 mtime.tv_sec = 0;
2776 mtime.tv_nsec = 0;
2777 cs_mtime.tv_sec = 0;
2778 cs_mtime.tv_nsec = 0;
2779
2780 /* get file's pathname and/or filename */
2781 pathname = NULL;
2782 filename = NULL;
2783 pathname_len = 0;
2784 filename_len = 0;
2785 truncated_path = FALSE;
2786 /* no pager -> no file -> no pathname, use "<nil>" in that case */
2787 if (file_object->pager != NULL) {
2788 pathname = (char *)kalloc(__PATH_MAX * 2);
2789 if (pathname) {
2790 pathname[0] = '\0';
2791 pathname_len = __PATH_MAX;
2792 filename = pathname + pathname_len;
2793 filename_len = __PATH_MAX;
2794 }
2795 vnode_pager_get_object_name(file_object->pager,
2796 pathname,
2797 pathname_len,
2798 filename,
2799 filename_len,
2800 &truncated_path);
2801 if (pathname) {
2802 /* safety first... */
2803 pathname[__PATH_MAX-1] = '\0';
2804 filename[__PATH_MAX-1] = '\0';
2805 }
2806 vnode_pager_get_object_mtime(file_object->pager,
2807 &mtime,
2808 &cs_mtime);
2809 }
2810 printf("CODE SIGNING: process %d[%s]: "
2811 "rejecting invalid page at address 0x%llx "
2812 "from offset 0x%llx in file \"%s%s%s\" "
2813 "(cs_mtime:%lu.%ld %s mtime:%lu.%ld) "
2814 "(signed:%d validated:%d tainted:%d nx:%d "
2815 "wpmapped:%d slid:%d dirty:%d depth:%d)\n",
2816 pid, procname, (addr64_t) vaddr,
2817 file_offset,
2818 (pathname ? pathname : "<nil>"),
2819 (truncated_path ? "/.../" : ""),
2820 (truncated_path ? filename : ""),
2821 cs_mtime.tv_sec, cs_mtime.tv_nsec,
2822 ((cs_mtime.tv_sec == mtime.tv_sec &&
2823 cs_mtime.tv_nsec == mtime.tv_nsec)
2824 ? "=="
2825 : "!="),
2826 mtime.tv_sec, mtime.tv_nsec,
2827 object->code_signed,
2828 m->cs_validated,
2829 m->cs_tainted,
2830 m->cs_nx,
2831 m->wpmapped,
2832 m->slid,
2833 m->dirty,
2834 shadow_depth);
2835
2836 /*
2837 * We currently only generate an exit reason if cs_invalid_page directly killed a process. If cs_invalid_page
2838 * did not kill the process (more the case on desktop), vm_fault_enter will not satisfy the fault and whether the
2839 * process dies is dependent on whether there is a signal handler registered for SIGSEGV and how that handler
2840 * will deal with the segmentation fault.
2841 */
2842 if (cs_killed) {
2843 KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXITREASON_CREATE) | DBG_FUNC_NONE,
2844 pid, OS_REASON_CODESIGNING, CODESIGNING_EXIT_REASON_INVALID_PAGE, 0, 0);
2845
2846 codesigning_exit_reason = os_reason_create(OS_REASON_CODESIGNING, CODESIGNING_EXIT_REASON_INVALID_PAGE);
2847 if (codesigning_exit_reason == NULL) {
2848 printf("vm_fault_enter: failed to allocate codesigning exit reason\n");
2849 } else {
2850 mach_vm_address_t data_addr = 0;
2851 struct codesigning_exit_reason_info *ceri = NULL;
2852 uint32_t reason_buffer_size_estimate = kcdata_estimate_required_buffer_size(1, sizeof(*ceri));
2853
2854 if (os_reason_alloc_buffer_noblock(codesigning_exit_reason, reason_buffer_size_estimate)) {
2855 printf("vm_fault_enter: failed to allocate buffer for codesigning exit reason\n");
2856 } else {
2857 if (KERN_SUCCESS == kcdata_get_memory_addr(&codesigning_exit_reason->osr_kcd_descriptor,
2858 EXIT_REASON_CODESIGNING_INFO, sizeof(*ceri), &data_addr)) {
2859 ceri = (struct codesigning_exit_reason_info *)data_addr;
2860 static_assert(__PATH_MAX == sizeof(ceri->ceri_pathname));
2861
2862 ceri->ceri_virt_addr = vaddr;
2863 ceri->ceri_file_offset = file_offset;
2864 if (pathname)
2865 strncpy((char *)&ceri->ceri_pathname, pathname, sizeof(ceri->ceri_pathname));
2866 else
2867 ceri->ceri_pathname[0] = '\0';
2868 if (filename)
2869 strncpy((char *)&ceri->ceri_filename, filename, sizeof(ceri->ceri_filename));
2870 else
2871 ceri->ceri_filename[0] = '\0';
2872 ceri->ceri_path_truncated = (truncated_path);
2873 ceri->ceri_codesig_modtime_secs = cs_mtime.tv_sec;
2874 ceri->ceri_codesig_modtime_nsecs = cs_mtime.tv_nsec;
2875 ceri->ceri_page_modtime_secs = mtime.tv_sec;
2876 ceri->ceri_page_modtime_nsecs = mtime.tv_nsec;
2877 ceri->ceri_object_codesigned = (object->code_signed);
2878 ceri->ceri_page_codesig_validated = (m->cs_validated);
2879 ceri->ceri_page_codesig_tainted = (m->cs_tainted);
2880 ceri->ceri_page_codesig_nx = (m->cs_nx);
2881 ceri->ceri_page_wpmapped = (m->wpmapped);
2882 ceri->ceri_page_slid = (m->slid);
2883 ceri->ceri_page_dirty = (m->dirty);
2884 ceri->ceri_page_shadow_depth = shadow_depth;
2885 } else {
2886 #if DEBUG || DEVELOPMENT
2887 panic("vm_fault_enter: failed to allocate kcdata for codesigning exit reason");
2888 #else
2889 printf("vm_fault_enter: failed to allocate kcdata for codesigning exit reason\n");
2890 #endif /* DEBUG || DEVELOPMENT */
2891 /* Free the buffer */
2892 os_reason_alloc_buffer_noblock(codesigning_exit_reason, 0);
2893 }
2894 }
2895 }
2896
2897 set_thread_exit_reason(current_thread(), codesigning_exit_reason, FALSE);
2898 }
2899 if (panic_on_cs_killed &&
2900 object->object_slid) {
2901 panic("CODE SIGNING: process %d[%s]: "
2902 "rejecting invalid page at address 0x%llx "
2903 "from offset 0x%llx in file \"%s%s%s\" "
2904 "(cs_mtime:%lu.%ld %s mtime:%lu.%ld) "
2905 "(signed:%d validated:%d tainted:%d nx:%d"
2906 "wpmapped:%d slid:%d dirty:%d depth:%d)\n",
2907 pid, procname, (addr64_t) vaddr,
2908 file_offset,
2909 (pathname ? pathname : "<nil>"),
2910 (truncated_path ? "/.../" : ""),
2911 (truncated_path ? filename : ""),
2912 cs_mtime.tv_sec, cs_mtime.tv_nsec,
2913 ((cs_mtime.tv_sec == mtime.tv_sec &&
2914 cs_mtime.tv_nsec == mtime.tv_nsec)
2915 ? "=="
2916 : "!="),
2917 mtime.tv_sec, mtime.tv_nsec,
2918 object->code_signed,
2919 m->cs_validated,
2920 m->cs_tainted,
2921 m->cs_nx,
2922 m->wpmapped,
2923 m->slid,
2924 m->dirty,
2925 shadow_depth);
2926 }
2927
2928 if (file_object != object) {
2929 vm_object_unlock(file_object);
2930 }
2931 if (pathname_len != 0) {
2932 kfree(pathname, __PATH_MAX * 2);
2933 pathname = NULL;
2934 filename = NULL;
2935 }
2936 } else {
2937 /* proceed with the invalid page */
2938 kr = KERN_SUCCESS;
2939 if (!m->cs_validated &&
2940 !object->code_signed) {
2941 /*
2942 * This page has not been (fully) validated but
2943 * does not belong to a code-signed object
2944 * so it should not be forcefully considered
2945 * as tainted.
2946 * We're just concerned about it here because
2947 * we've been asked to "execute" it but that
2948 * does not mean that it should cause other
2949 * accesses to fail.
2950 * This happens when a debugger sets a
2951 * breakpoint and we then execute code in
2952 * that page. Marking the page as "tainted"
2953 * would cause any inspection tool ("leaks",
2954 * "vmmap", "CrashReporter", ...) to get killed
2955 * due to code-signing violation on that page,
2956 * even though they're just reading it and not
2957 * executing from it.
2958 */
2959 } else {
2960 /*
2961 * Page might have been tainted before or not;
2962 * now it definitively is. If the page wasn't
2963 * tainted, we must disconnect it from all
2964 * pmaps later, to force existing mappings
2965 * through that code path for re-consideration
2966 * of the validity of that page.
2967 */
2968 must_disconnect = !m->cs_tainted;
2969 m->cs_tainted = TRUE;
2970 }
2971 cs_enter_tainted_accepted++;
2972 }
2973 if (kr != KERN_SUCCESS) {
2974 if (cs_debug) {
2975 printf("CODESIGNING: vm_fault_enter(0x%llx): "
2976 "*** INVALID PAGE ***\n",
2977 (long long)vaddr);
2978 }
2979 #if !SECURE_KERNEL
2980 if (cs_enforcement_panic) {
2981 panic("CODESIGNING: panicking on invalid page\n");
2982 }
2983 #endif
2984 }
2985
2986 } else {
2987 /* proceed with the valid page */
2988 kr = KERN_SUCCESS;
2989 }
2990
2991 boolean_t page_queues_locked = FALSE;
2992 #define __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED() \
2993 MACRO_BEGIN \
2994 if (! page_queues_locked) { \
2995 page_queues_locked = TRUE; \
2996 vm_page_lockspin_queues(); \
2997 } \
2998 MACRO_END
2999 #define __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED() \
3000 MACRO_BEGIN \
3001 if (page_queues_locked) { \
3002 page_queues_locked = FALSE; \
3003 vm_page_unlock_queues(); \
3004 } \
3005 MACRO_END
3006
3007 /*
3008 * Hold queues lock to manipulate
3009 * the page queues. Change wiring
3010 * case is obvious.
3011 */
3012 assert((m->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) || object != compressor_object);
3013
3014 #if CONFIG_BACKGROUND_QUEUE
3015 vm_page_update_background_state(m);
3016 #endif
3017 if (m->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
3018 /*
3019 * Compressor pages are neither wired
3020 * nor pageable and should never change.
3021 */
3022 assert(object == compressor_object);
3023 } else if (change_wiring) {
3024 __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED();
3025
3026 if (wired) {
3027 if (kr == KERN_SUCCESS) {
3028 vm_page_wire(m, VM_PROT_MEMORY_TAG(caller_prot), TRUE);
3029 }
3030 } else {
3031 vm_page_unwire(m, TRUE);
3032 }
3033 /* we keep the page queues lock, if we need it later */
3034
3035 } else {
3036 if (object->internal == TRUE) {
3037 /*
3038 * don't allow anonymous pages on
3039 * the speculative queues
3040 */
3041 no_cache = FALSE;
3042 }
3043 if (kr != KERN_SUCCESS) {
3044 __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED();
3045 vm_page_deactivate(m);
3046 /* we keep the page queues lock, if we need it later */
3047 } else if (((m->vm_page_q_state == VM_PAGE_NOT_ON_Q) ||
3048 (m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) ||
3049 (m->vm_page_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) ||
3050 ((m->vm_page_q_state != VM_PAGE_ON_THROTTLED_Q) && no_cache)) &&
3051 !VM_PAGE_WIRED(m)) {
3052
3053 if (vm_page_local_q &&
3054 (*type_of_fault == DBG_COW_FAULT ||
3055 *type_of_fault == DBG_ZERO_FILL_FAULT) ) {
3056 struct vpl *lq;
3057 uint32_t lid;
3058
3059 assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
3060
3061 __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED();
3062 vm_object_lock_assert_exclusive(object);
3063
3064 /*
3065 * we got a local queue to stuff this
3066 * new page on...
3067 * its safe to manipulate local and
3068 * local_id at this point since we're
3069 * behind an exclusive object lock and
3070 * the page is not on any global queue.
3071 *
3072 * we'll use the current cpu number to
3073 * select the queue note that we don't
3074 * need to disable preemption... we're
3075 * going to be behind the local queue's
3076 * lock to do the real work
3077 */
3078 lid = cpu_number();
3079
3080 lq = &vm_page_local_q[lid].vpl_un.vpl;
3081
3082 VPL_LOCK(&lq->vpl_lock);
3083
3084 vm_page_check_pageable_safe(m);
3085 vm_page_queue_enter(&lq->vpl_queue, m,
3086 vm_page_t, pageq);
3087 m->vm_page_q_state = VM_PAGE_ON_ACTIVE_LOCAL_Q;
3088 m->local_id = lid;
3089 lq->vpl_count++;
3090
3091 if (object->internal)
3092 lq->vpl_internal_count++;
3093 else
3094 lq->vpl_external_count++;
3095
3096 VPL_UNLOCK(&lq->vpl_lock);
3097
3098 if (lq->vpl_count > vm_page_local_q_soft_limit)
3099 {
3100 /*
3101 * we're beyond the soft limit
3102 * for the local queue
3103 * vm_page_reactivate_local will
3104 * 'try' to take the global page
3105 * queue lock... if it can't
3106 * that's ok... we'll let the
3107 * queue continue to grow up
3108 * to the hard limit... at that
3109 * point we'll wait for the
3110 * lock... once we've got the
3111 * lock, we'll transfer all of
3112 * the pages from the local
3113 * queue to the global active
3114 * queue
3115 */
3116 vm_page_reactivate_local(lid, FALSE, FALSE);
3117 }
3118 } else {
3119
3120 __VM_PAGE_LOCKSPIN_QUEUES_IF_NEEDED();
3121
3122 /*
3123 * test again now that we hold the
3124 * page queue lock
3125 */
3126 if (!VM_PAGE_WIRED(m)) {
3127 if (m->vm_page_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
3128 vm_page_queues_remove(m, FALSE);
3129
3130 vm_pageout_cleaned_reactivated++;
3131 vm_pageout_cleaned_fault_reactivated++;
3132 }
3133
3134 if ( !VM_PAGE_ACTIVE_OR_INACTIVE(m) ||
3135 no_cache) {
3136 /*
3137 * If this is a no_cache mapping
3138 * and the page has never been
3139 * mapped before or was
3140 * previously a no_cache page,
3141 * then we want to leave pages
3142 * in the speculative state so
3143 * that they can be readily
3144 * recycled if free memory runs
3145 * low. Otherwise the page is
3146 * activated as normal.
3147 */
3148
3149 if (no_cache &&
3150 (!previously_pmapped ||
3151 m->no_cache)) {
3152 m->no_cache = TRUE;
3153
3154 if (m->vm_page_q_state != VM_PAGE_ON_SPECULATIVE_Q)
3155 vm_page_speculate(m, FALSE);
3156
3157 } else if ( !VM_PAGE_ACTIVE_OR_INACTIVE(m)) {
3158 vm_page_activate(m);
3159 }
3160 }
3161 }
3162 /* we keep the page queues lock, if we need it later */
3163 }
3164 }
3165 }
3166 /* we're done with the page queues lock, if we ever took it */
3167 __VM_PAGE_UNLOCK_QUEUES_IF_NEEDED();
3168
3169
3170 /* If we have a KERN_SUCCESS from the previous checks, we either have
3171 * a good page, or a tainted page that has been accepted by the process.
3172 * In both cases the page will be entered into the pmap.
3173 * If the page is writeable, we need to disconnect it from other pmaps
3174 * now so those processes can take note.
3175 */
3176 if (kr == KERN_SUCCESS) {
3177 /*
3178 * NOTE: we may only hold the vm_object lock SHARED
3179 * at this point, so we need the phys_page lock to
3180 * properly serialize updating the pmapped and
3181 * xpmapped bits
3182 */
3183 if ((prot & VM_PROT_EXECUTE) && !m->xpmapped) {
3184 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
3185
3186 pmap_lock_phys_page(phys_page);
3187 /*
3188 * go ahead and take the opportunity
3189 * to set 'pmapped' here so that we don't
3190 * need to grab this lock a 2nd time
3191 * just below
3192 */
3193 m->pmapped = TRUE;
3194
3195 if (!m->xpmapped) {
3196
3197 m->xpmapped = TRUE;
3198
3199 pmap_unlock_phys_page(phys_page);
3200
3201 if (!object->internal)
3202 OSAddAtomic(1, &vm_page_xpmapped_external_count);
3203
3204 if (object->internal &&
3205 object->pager != NULL) {
3206 /*
3207 * This page could have been
3208 * uncompressed by the
3209 * compressor pager and its
3210 * contents might be only in
3211 * the data cache.
3212 * Since it's being mapped for
3213 * "execute" for the fist time,
3214 * make sure the icache is in
3215 * sync.
3216 */
3217 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
3218 pmap_sync_page_data_phys(phys_page);
3219 }
3220 } else
3221 pmap_unlock_phys_page(phys_page);
3222 } else {
3223 if (m->pmapped == FALSE) {
3224 ppnum_t phys_page = VM_PAGE_GET_PHYS_PAGE(m);
3225
3226 pmap_lock_phys_page(phys_page);
3227 m->pmapped = TRUE;
3228 pmap_unlock_phys_page(phys_page);
3229 }
3230 }
3231 if (vm_page_is_slideable(m)) {
3232 boolean_t was_busy = m->busy;
3233
3234 vm_object_lock_assert_exclusive(object);
3235
3236 m->busy = TRUE;
3237 kr = vm_page_slide(m, 0);
3238 assert(m->busy);
3239 if(!was_busy) {
3240 PAGE_WAKEUP_DONE(m);
3241 }
3242 if (kr != KERN_SUCCESS) {
3243 /*
3244 * This page has not been slid correctly,
3245 * do not do the pmap_enter() !
3246 * Let vm_fault_enter() return the error
3247 * so the caller can fail the fault.
3248 */
3249 goto after_the_pmap_enter;
3250 }
3251 }
3252
3253 if (fault_type & VM_PROT_WRITE) {
3254
3255 if (m->wpmapped == FALSE) {
3256 vm_object_lock_assert_exclusive(object);
3257 if (!object->internal && object->pager) {
3258 task_update_logical_writes(current_task(), PAGE_SIZE, TASK_WRITE_DEFERRED, vnode_pager_lookup_vnode(object->pager));
3259 }
3260 m->wpmapped = TRUE;
3261 }
3262 if (must_disconnect) {
3263 /*
3264 * We can only get here
3265 * because of the CSE logic
3266 */
3267 assert(cs_enforcement_enabled);
3268 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
3269 /*
3270 * If we are faulting for a write, we can clear
3271 * the execute bit - that will ensure the page is
3272 * checked again before being executable, which
3273 * protects against a map switch.
3274 * This only happens the first time the page
3275 * gets tainted, so we won't get stuck here
3276 * to make an already writeable page executable.
3277 */
3278 if (!cs_bypass){
3279 prot &= ~VM_PROT_EXECUTE;
3280 }
3281 }
3282 }
3283 assert(VM_PAGE_OBJECT(m) == object);
3284
3285 /* Prevent a deadlock by not
3286 * holding the object lock if we need to wait for a page in
3287 * pmap_enter() - <rdar://problem/7138958> */
3288 PMAP_ENTER_OPTIONS(pmap, vaddr, m, prot, fault_type, 0,
3289 wired,
3290 pmap_options | PMAP_OPTIONS_NOWAIT,
3291 pe_result);
3292
3293 if(pe_result == KERN_RESOURCE_SHORTAGE) {
3294
3295 if (need_retry) {
3296 /*
3297 * this will be non-null in the case where we hold the lock
3298 * on the top-object in this chain... we can't just drop
3299 * the lock on the object we're inserting the page into
3300 * and recall the PMAP_ENTER since we can still cause
3301 * a deadlock if one of the critical paths tries to
3302 * acquire the lock on the top-object and we're blocked
3303 * in PMAP_ENTER waiting for memory... our only recourse
3304 * is to deal with it at a higher level where we can
3305 * drop both locks.
3306 */
3307 *need_retry = TRUE;
3308 vm_pmap_enter_retried++;
3309 goto after_the_pmap_enter;
3310 }
3311 /* The nonblocking version of pmap_enter did not succeed.
3312 * and we don't need to drop other locks and retry
3313 * at the level above us, so
3314 * use the blocking version instead. Requires marking
3315 * the page busy and unlocking the object */
3316 boolean_t was_busy = m->busy;
3317
3318 vm_object_lock_assert_exclusive(object);
3319
3320 m->busy = TRUE;
3321 vm_object_unlock(object);
3322
3323 PMAP_ENTER_OPTIONS(pmap, vaddr, m, prot, fault_type,
3324 0, wired,
3325 pmap_options, pe_result);
3326
3327 assert(VM_PAGE_OBJECT(m) == object);
3328
3329 /* Take the object lock again. */
3330 vm_object_lock(object);
3331
3332 /* If the page was busy, someone else will wake it up.
3333 * Otherwise, we have to do it now. */
3334 assert(m->busy);
3335 if(!was_busy) {
3336 PAGE_WAKEUP_DONE(m);
3337 }
3338 vm_pmap_enter_blocked++;
3339 }
3340 }
3341
3342 after_the_pmap_enter:
3343 return kr;
3344 }
3345
3346 void
3347 vm_pre_fault(vm_map_offset_t vaddr)
3348 {
3349 if (pmap_find_phys(current_map()->pmap, vaddr) == 0) {
3350
3351 vm_fault(current_map(), /* map */
3352 vaddr, /* vaddr */
3353 VM_PROT_READ, /* fault_type */
3354 FALSE, /* change_wiring */
3355 THREAD_UNINT, /* interruptible */
3356 NULL, /* caller_pmap */
3357 0 /* caller_pmap_addr */);
3358 }
3359 }
3360
3361
3362 /*
3363 * Routine: vm_fault
3364 * Purpose:
3365 * Handle page faults, including pseudo-faults
3366 * used to change the wiring status of pages.
3367 * Returns:
3368 * Explicit continuations have been removed.
3369 * Implementation:
3370 * vm_fault and vm_fault_page save mucho state
3371 * in the moral equivalent of a closure. The state
3372 * structure is allocated when first entering vm_fault
3373 * and deallocated when leaving vm_fault.
3374 */
3375
3376 extern int _map_enter_debug;
3377 extern uint64_t get_current_unique_pid(void);
3378
3379 unsigned long vm_fault_collapse_total = 0;
3380 unsigned long vm_fault_collapse_skipped = 0;
3381
3382
3383 kern_return_t
3384 vm_fault(
3385 vm_map_t map,
3386 vm_map_offset_t vaddr,
3387 vm_prot_t fault_type,
3388 boolean_t change_wiring,
3389 int interruptible,
3390 pmap_t caller_pmap,
3391 vm_map_offset_t caller_pmap_addr)
3392 {
3393 return vm_fault_internal(map, vaddr, fault_type, change_wiring,
3394 interruptible, caller_pmap, caller_pmap_addr,
3395 NULL);
3396 }
3397
3398
3399 kern_return_t
3400 vm_fault_internal(
3401 vm_map_t map,
3402 vm_map_offset_t vaddr,
3403 vm_prot_t caller_prot,
3404 boolean_t change_wiring,
3405 int interruptible,
3406 pmap_t caller_pmap,
3407 vm_map_offset_t caller_pmap_addr,
3408 ppnum_t *physpage_p)
3409 {
3410 vm_map_version_t version; /* Map version for verificiation */
3411 boolean_t wired; /* Should mapping be wired down? */
3412 vm_object_t object; /* Top-level object */
3413 vm_object_offset_t offset; /* Top-level offset */
3414 vm_prot_t prot; /* Protection for mapping */
3415 vm_object_t old_copy_object; /* Saved copy object */
3416 vm_page_t result_page; /* Result of vm_fault_page */
3417 vm_page_t top_page; /* Placeholder page */
3418 kern_return_t kr;
3419
3420 vm_page_t m; /* Fast access to result_page */
3421 kern_return_t error_code;
3422 vm_object_t cur_object;
3423 vm_object_t m_object = NULL;
3424 vm_object_offset_t cur_offset;
3425 vm_page_t cur_m;
3426 vm_object_t new_object;
3427 int type_of_fault;
3428 pmap_t pmap;
3429 boolean_t interruptible_state;
3430 vm_map_t real_map = map;
3431 vm_map_t original_map = map;
3432 vm_prot_t fault_type;
3433 vm_prot_t original_fault_type;
3434 struct vm_object_fault_info fault_info;
3435 boolean_t need_collapse = FALSE;
3436 boolean_t need_retry = FALSE;
3437 boolean_t *need_retry_ptr = NULL;
3438 int object_lock_type = 0;
3439 int cur_object_lock_type;
3440 vm_object_t top_object = VM_OBJECT_NULL;
3441 int throttle_delay;
3442 int compressed_count_delta;
3443 int grab_options;
3444 vm_map_offset_t trace_vaddr;
3445 vm_map_offset_t trace_real_vaddr;
3446 #if DEVELOPMENT || DEBUG
3447 vm_map_offset_t real_vaddr;
3448
3449 real_vaddr = vaddr;
3450 #endif /* DEVELOPMENT || DEBUG */
3451 trace_real_vaddr = vaddr;
3452 vaddr = vm_map_trunc_page(vaddr, PAGE_MASK);
3453
3454 if (map == kernel_map) {
3455 trace_vaddr = VM_KERNEL_UNSLIDE_OR_PERM(vaddr);
3456 trace_real_vaddr = VM_KERNEL_UNSLIDE_OR_PERM(trace_real_vaddr);
3457 } else {
3458 trace_vaddr = vaddr;
3459 }
3460
3461 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
3462 (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_START,
3463 ((uint64_t)trace_vaddr >> 32),
3464 trace_vaddr,
3465 (map == kernel_map),
3466 0,
3467 0);
3468
3469 if (get_preemption_level() != 0) {
3470 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
3471 (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END,
3472 ((uint64_t)trace_vaddr >> 32),
3473 trace_vaddr,
3474 KERN_FAILURE,
3475 0,
3476 0);
3477
3478 return (KERN_FAILURE);
3479 }
3480
3481 interruptible_state = thread_interrupt_level(interruptible);
3482
3483 fault_type = (change_wiring ? VM_PROT_NONE : caller_prot);
3484
3485 VM_STAT_INCR(faults);
3486 current_task()->faults++;
3487 original_fault_type = fault_type;
3488
3489 if (fault_type & VM_PROT_WRITE)
3490 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3491 else
3492 object_lock_type = OBJECT_LOCK_SHARED;
3493
3494 cur_object_lock_type = OBJECT_LOCK_SHARED;
3495
3496 RetryFault:
3497 /*
3498 * assume we will hit a page in the cache
3499 * otherwise, explicitly override with
3500 * the real fault type once we determine it
3501 */
3502 type_of_fault = DBG_CACHE_HIT_FAULT;
3503
3504 /*
3505 * Find the backing store object and offset into
3506 * it to begin the search.
3507 */
3508 fault_type = original_fault_type;
3509 map = original_map;
3510 vm_map_lock_read(map);
3511
3512 kr = vm_map_lookup_locked(&map, vaddr, fault_type,
3513 object_lock_type, &version,
3514 &object, &offset, &prot, &wired,
3515 &fault_info,
3516 &real_map);
3517
3518
3519 if (kr != KERN_SUCCESS) {
3520 vm_map_unlock_read(map);
3521 goto done;
3522 }
3523 pmap = real_map->pmap;
3524 fault_info.interruptible = interruptible;
3525 fault_info.stealth = FALSE;
3526 fault_info.io_sync = FALSE;
3527 fault_info.mark_zf_absent = FALSE;
3528 fault_info.batch_pmap_op = FALSE;
3529
3530 /*
3531 * If the page is wired, we must fault for the current protection
3532 * value, to avoid further faults.
3533 */
3534 if (wired) {
3535 fault_type = prot | VM_PROT_WRITE;
3536 /*
3537 * since we're treating this fault as a 'write'
3538 * we must hold the top object lock exclusively
3539 */
3540 if (object_lock_type == OBJECT_LOCK_SHARED) {
3541
3542 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3543
3544 if (vm_object_lock_upgrade(object) == FALSE) {
3545 /*
3546 * couldn't upgrade, so explictly
3547 * take the lock exclusively
3548 */
3549 vm_object_lock(object);
3550 }
3551 }
3552 }
3553
3554 #if VM_FAULT_CLASSIFY
3555 /*
3556 * Temporary data gathering code
3557 */
3558 vm_fault_classify(object, offset, fault_type);
3559 #endif
3560 /*
3561 * Fast fault code. The basic idea is to do as much as
3562 * possible while holding the map lock and object locks.
3563 * Busy pages are not used until the object lock has to
3564 * be dropped to do something (copy, zero fill, pmap enter).
3565 * Similarly, paging references aren't acquired until that
3566 * point, and object references aren't used.
3567 *
3568 * If we can figure out what to do
3569 * (zero fill, copy on write, pmap enter) while holding
3570 * the locks, then it gets done. Otherwise, we give up,
3571 * and use the original fault path (which doesn't hold
3572 * the map lock, and relies on busy pages).
3573 * The give up cases include:
3574 * - Have to talk to pager.
3575 * - Page is busy, absent or in error.
3576 * - Pager has locked out desired access.
3577 * - Fault needs to be restarted.
3578 * - Have to push page into copy object.
3579 *
3580 * The code is an infinite loop that moves one level down
3581 * the shadow chain each time. cur_object and cur_offset
3582 * refer to the current object being examined. object and offset
3583 * are the original object from the map. The loop is at the
3584 * top level if and only if object and cur_object are the same.
3585 *
3586 * Invariants: Map lock is held throughout. Lock is held on
3587 * original object and cur_object (if different) when
3588 * continuing or exiting loop.
3589 *
3590 */
3591
3592
3593 /*
3594 * If this page is to be inserted in a copy delay object
3595 * for writing, and if the object has a copy, then the
3596 * copy delay strategy is implemented in the slow fault page.
3597 */
3598 if (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY &&
3599 object->copy != VM_OBJECT_NULL && (fault_type & VM_PROT_WRITE))
3600 goto handle_copy_delay;
3601
3602 cur_object = object;
3603 cur_offset = offset;
3604
3605 grab_options = 0;
3606 #if CONFIG_SECLUDED_MEMORY
3607 if (object->can_grab_secluded) {
3608 grab_options |= VM_PAGE_GRAB_SECLUDED;
3609 }
3610 #endif /* CONFIG_SECLUDED_MEMORY */
3611
3612 while (TRUE) {
3613 if (!cur_object->pager_created &&
3614 cur_object->phys_contiguous) /* superpage */
3615 break;
3616
3617 if (cur_object->blocked_access) {
3618 /*
3619 * Access to this VM object has been blocked.
3620 * Let the slow path handle it.
3621 */
3622 break;
3623 }
3624
3625 m = vm_page_lookup(cur_object, cur_offset);
3626 m_object = NULL;
3627
3628 if (m != VM_PAGE_NULL) {
3629 m_object = cur_object;
3630
3631 if (m->busy) {
3632 wait_result_t result;
3633
3634 /*
3635 * in order to do the PAGE_ASSERT_WAIT, we must
3636 * have object that 'm' belongs to locked exclusively
3637 */
3638 if (object != cur_object) {
3639
3640 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3641
3642 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3643
3644 if (vm_object_lock_upgrade(cur_object) == FALSE) {
3645 /*
3646 * couldn't upgrade so go do a full retry
3647 * immediately since we can no longer be
3648 * certain about cur_object (since we
3649 * don't hold a reference on it)...
3650 * first drop the top object lock
3651 */
3652 vm_object_unlock(object);
3653
3654 vm_map_unlock_read(map);
3655 if (real_map != map)
3656 vm_map_unlock(real_map);
3657
3658 goto RetryFault;
3659 }
3660 }
3661 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3662
3663 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3664
3665 if (vm_object_lock_upgrade(object) == FALSE) {
3666 /*
3667 * couldn't upgrade, so explictly take the lock
3668 * exclusively and go relookup the page since we
3669 * will have dropped the object lock and
3670 * a different thread could have inserted
3671 * a page at this offset
3672 * no need for a full retry since we're
3673 * at the top level of the object chain
3674 */
3675 vm_object_lock(object);
3676
3677 continue;
3678 }
3679 }
3680 if ((m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) && m_object->internal) {
3681 /*
3682 * m->busy == TRUE and the object is locked exclusively
3683 * if m->pageout_queue == TRUE after we acquire the
3684 * queues lock, we are guaranteed that it is stable on
3685 * the pageout queue and therefore reclaimable
3686 *
3687 * NOTE: this is only true for the internal pageout queue
3688 * in the compressor world
3689 */
3690 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
3691
3692 vm_page_lock_queues();
3693
3694 if (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) {
3695 vm_pageout_throttle_up(m);
3696 vm_page_unlock_queues();
3697
3698 PAGE_WAKEUP_DONE(m);
3699 goto reclaimed_from_pageout;
3700 }
3701 vm_page_unlock_queues();
3702 }
3703 if (object != cur_object)
3704 vm_object_unlock(object);
3705
3706 vm_map_unlock_read(map);
3707 if (real_map != map)
3708 vm_map_unlock(real_map);
3709
3710 result = PAGE_ASSERT_WAIT(m, interruptible);
3711
3712 vm_object_unlock(cur_object);
3713
3714 if (result == THREAD_WAITING) {
3715 result = thread_block(THREAD_CONTINUE_NULL);
3716
3717 counter(c_vm_fault_page_block_busy_kernel++);
3718 }
3719 if (result == THREAD_AWAKENED || result == THREAD_RESTART)
3720 goto RetryFault;
3721
3722 kr = KERN_ABORTED;
3723 goto done;
3724 }
3725 reclaimed_from_pageout:
3726 if (m->laundry) {
3727 if (object != cur_object) {
3728 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3729 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3730
3731 vm_object_unlock(object);
3732 vm_object_unlock(cur_object);
3733
3734 vm_map_unlock_read(map);
3735 if (real_map != map)
3736 vm_map_unlock(real_map);
3737
3738 goto RetryFault;
3739 }
3740
3741 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3742
3743 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3744
3745 if (vm_object_lock_upgrade(object) == FALSE) {
3746 /*
3747 * couldn't upgrade, so explictly take the lock
3748 * exclusively and go relookup the page since we
3749 * will have dropped the object lock and
3750 * a different thread could have inserted
3751 * a page at this offset
3752 * no need for a full retry since we're
3753 * at the top level of the object chain
3754 */
3755 vm_object_lock(object);
3756
3757 continue;
3758 }
3759 }
3760 vm_pageout_steal_laundry(m, FALSE);
3761 }
3762
3763 if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
3764 /*
3765 * Guard page: let the slow path deal with it
3766 */
3767 break;
3768 }
3769 if (m->unusual && (m->error || m->restart || m->private || m->absent)) {
3770 /*
3771 * Unusual case... let the slow path deal with it
3772 */
3773 break;
3774 }
3775 if (VM_OBJECT_PURGEABLE_FAULT_ERROR(m_object)) {
3776 if (object != cur_object)
3777 vm_object_unlock(object);
3778 vm_map_unlock_read(map);
3779 if (real_map != map)
3780 vm_map_unlock(real_map);
3781 vm_object_unlock(cur_object);
3782 kr = KERN_MEMORY_ERROR;
3783 goto done;
3784 }
3785
3786 if (m->encrypted) {
3787 /*
3788 * ENCRYPTED SWAP:
3789 * We've soft-faulted (because it's not in the page
3790 * table) on an encrypted page.
3791 * Keep the page "busy" so that no one messes with
3792 * it during the decryption.
3793 * Release the extra locks we're holding, keep only
3794 * the page's VM object lock.
3795 *
3796 * in order to set 'busy' on 'm', we must
3797 * have object that 'm' belongs to locked exclusively
3798 */
3799 if (object != cur_object) {
3800 vm_object_unlock(object);
3801
3802 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3803
3804 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3805
3806 if (vm_object_lock_upgrade(cur_object) == FALSE) {
3807 /*
3808 * couldn't upgrade so go do a full retry
3809 * immediately since we've already dropped
3810 * the top object lock associated with this page
3811 * and the current one got dropped due to the
3812 * failed upgrade... the state is no longer valid
3813 */
3814 vm_map_unlock_read(map);
3815 if (real_map != map)
3816 vm_map_unlock(real_map);
3817
3818 goto RetryFault;
3819 }
3820 }
3821 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3822
3823 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3824
3825 if (vm_object_lock_upgrade(object) == FALSE) {
3826 /*
3827 * couldn't upgrade, so explictly take the lock
3828 * exclusively and go relookup the page since we
3829 * will have dropped the object lock and
3830 * a different thread could have inserted
3831 * a page at this offset
3832 * no need for a full retry since we're
3833 * at the top level of the object chain
3834 */
3835 vm_object_lock(object);
3836
3837 continue;
3838 }
3839 }
3840 m->busy = TRUE;
3841
3842 vm_map_unlock_read(map);
3843 if (real_map != map)
3844 vm_map_unlock(real_map);
3845
3846 vm_page_decrypt(m, 0);
3847
3848 assert(m->busy);
3849 PAGE_WAKEUP_DONE(m);
3850
3851 vm_object_unlock(cur_object);
3852 /*
3853 * Retry from the top, in case anything
3854 * changed while we were decrypting...
3855 */
3856 goto RetryFault;
3857 }
3858 ASSERT_PAGE_DECRYPTED(m);
3859
3860 if(vm_page_is_slideable(m)) {
3861 /*
3862 * We might need to slide this page, and so,
3863 * we want to hold the VM object exclusively.
3864 */
3865 if (object != cur_object) {
3866 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3867 vm_object_unlock(object);
3868 vm_object_unlock(cur_object);
3869
3870 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3871
3872 vm_map_unlock_read(map);
3873 if (real_map != map)
3874 vm_map_unlock(real_map);
3875
3876 goto RetryFault;
3877 }
3878 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3879
3880 vm_object_unlock(object);
3881 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3882 vm_map_unlock_read(map);
3883 goto RetryFault;
3884 }
3885 }
3886 assert(m_object == VM_PAGE_OBJECT(m));
3887
3888 if (VM_FAULT_NEED_CS_VALIDATION(map->pmap, m, m_object) ||
3889 (physpage_p != NULL && (prot & VM_PROT_WRITE))) {
3890 upgrade_for_validation:
3891 /*
3892 * We might need to validate this page
3893 * against its code signature, so we
3894 * want to hold the VM object exclusively.
3895 */
3896 if (object != cur_object) {
3897 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
3898 vm_object_unlock(object);
3899 vm_object_unlock(cur_object);
3900
3901 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3902
3903 vm_map_unlock_read(map);
3904 if (real_map != map)
3905 vm_map_unlock(real_map);
3906
3907 goto RetryFault;
3908 }
3909
3910 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
3911
3912 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
3913
3914 if (vm_object_lock_upgrade(object) == FALSE) {
3915 /*
3916 * couldn't upgrade, so explictly take the lock
3917 * exclusively and go relookup the page since we
3918 * will have dropped the object lock and
3919 * a different thread could have inserted
3920 * a page at this offset
3921 * no need for a full retry since we're
3922 * at the top level of the object chain
3923 */
3924 vm_object_lock(object);
3925
3926 continue;
3927 }
3928 }
3929 }
3930 /*
3931 * Two cases of map in faults:
3932 * - At top level w/o copy object.
3933 * - Read fault anywhere.
3934 * --> must disallow write.
3935 */
3936
3937 if (object == cur_object && object->copy == VM_OBJECT_NULL) {
3938
3939 goto FastPmapEnter;
3940 }
3941
3942 if ((fault_type & VM_PROT_WRITE) == 0) {
3943
3944 prot &= ~VM_PROT_WRITE;
3945
3946 if (object != cur_object) {
3947 /*
3948 * We still need to hold the top object
3949 * lock here to prevent a race between
3950 * a read fault (taking only "shared"
3951 * locks) and a write fault (taking
3952 * an "exclusive" lock on the top
3953 * object.
3954 * Otherwise, as soon as we release the
3955 * top lock, the write fault could
3956 * proceed and actually complete before
3957 * the read fault, and the copied page's
3958 * translation could then be overwritten
3959 * by the read fault's translation for
3960 * the original page.
3961 *
3962 * Let's just record what the top object
3963 * is and we'll release it later.
3964 */
3965 top_object = object;
3966
3967 /*
3968 * switch to the object that has the new page
3969 */
3970 object = cur_object;
3971 object_lock_type = cur_object_lock_type;
3972 }
3973 FastPmapEnter:
3974 assert(m_object == VM_PAGE_OBJECT(m));
3975
3976 /*
3977 * prepare for the pmap_enter...
3978 * object and map are both locked
3979 * m contains valid data
3980 * object == m->object
3981 * cur_object == NULL or it's been unlocked
3982 * no paging references on either object or cur_object
3983 */
3984 if (top_object != VM_OBJECT_NULL || object_lock_type != OBJECT_LOCK_EXCLUSIVE)
3985 need_retry_ptr = &need_retry;
3986 else
3987 need_retry_ptr = NULL;
3988
3989 if (caller_pmap) {
3990 kr = vm_fault_enter(m,
3991 caller_pmap,
3992 caller_pmap_addr,
3993 prot,
3994 caller_prot,
3995 wired,
3996 change_wiring,
3997 fault_info.no_cache,
3998 fault_info.cs_bypass,
3999 fault_info.user_tag,
4000 fault_info.pmap_options,
4001 need_retry_ptr,
4002 &type_of_fault);
4003 } else {
4004 kr = vm_fault_enter(m,
4005 pmap,
4006 vaddr,
4007 prot,
4008 caller_prot,
4009 wired,
4010 change_wiring,
4011 fault_info.no_cache,
4012 fault_info.cs_bypass,
4013 fault_info.user_tag,
4014 fault_info.pmap_options,
4015 need_retry_ptr,
4016 &type_of_fault);
4017 }
4018 #if DEVELOPMENT || DEBUG
4019 {
4020 int event_code = 0;
4021
4022 if (m_object->internal)
4023 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_INTERNAL));
4024 else if (m_object->object_slid)
4025 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_SHAREDCACHE));
4026 else
4027 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_EXTERNAL));
4028
4029 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, event_code, trace_real_vaddr, (fault_info.user_tag << 16) | (caller_prot << 8) | type_of_fault, m->offset, get_current_unique_pid(), 0);
4030
4031 DTRACE_VM6(real_fault, vm_map_offset_t, real_vaddr, vm_map_offset_t, m->offset, int, event_code, int, caller_prot, int, type_of_fault, int, fault_info.user_tag);
4032 }
4033 #endif
4034 if (kr == KERN_SUCCESS &&
4035 physpage_p != NULL) {
4036 /* for vm_map_wire_and_extract() */
4037 *physpage_p = VM_PAGE_GET_PHYS_PAGE(m);
4038 if (prot & VM_PROT_WRITE) {
4039 vm_object_lock_assert_exclusive(m_object);
4040 m->dirty = TRUE;
4041 }
4042 }
4043
4044 if (top_object != VM_OBJECT_NULL) {
4045 /*
4046 * It's safe to drop the top object
4047 * now that we've done our
4048 * vm_fault_enter(). Any other fault
4049 * in progress for that virtual
4050 * address will either find our page
4051 * and translation or put in a new page
4052 * and translation.
4053 */
4054 vm_object_unlock(top_object);
4055 top_object = VM_OBJECT_NULL;
4056 }
4057
4058 if (need_collapse == TRUE)
4059 vm_object_collapse(object, offset, TRUE);
4060
4061 if (need_retry == FALSE &&
4062 (type_of_fault == DBG_PAGEIND_FAULT || type_of_fault == DBG_PAGEINV_FAULT || type_of_fault == DBG_CACHE_HIT_FAULT)) {
4063 /*
4064 * evaluate access pattern and update state
4065 * vm_fault_deactivate_behind depends on the
4066 * state being up to date
4067 */
4068 vm_fault_is_sequential(object, cur_offset, fault_info.behavior);
4069
4070 vm_fault_deactivate_behind(object, cur_offset, fault_info.behavior);
4071 }
4072 /*
4073 * That's it, clean up and return.
4074 */
4075 if (m->busy)
4076 PAGE_WAKEUP_DONE(m);
4077
4078 vm_object_unlock(object);
4079
4080 vm_map_unlock_read(map);
4081 if (real_map != map)
4082 vm_map_unlock(real_map);
4083
4084 if (need_retry == TRUE) {
4085 /*
4086 * vm_fault_enter couldn't complete the PMAP_ENTER...
4087 * at this point we don't hold any locks so it's safe
4088 * to ask the pmap layer to expand the page table to
4089 * accommodate this mapping... once expanded, we'll
4090 * re-drive the fault which should result in vm_fault_enter
4091 * being able to successfully enter the mapping this time around
4092 */
4093 (void)pmap_enter_options(
4094 pmap, vaddr, 0, 0, 0, 0, 0,
4095 PMAP_OPTIONS_NOENTER, NULL);
4096
4097 need_retry = FALSE;
4098 goto RetryFault;
4099 }
4100 goto done;
4101 }
4102 /*
4103 * COPY ON WRITE FAULT
4104 */
4105 assert(object_lock_type == OBJECT_LOCK_EXCLUSIVE);
4106
4107 /*
4108 * If objects match, then
4109 * object->copy must not be NULL (else control
4110 * would be in previous code block), and we
4111 * have a potential push into the copy object
4112 * with which we can't cope with here.
4113 */
4114 if (cur_object == object) {
4115 /*
4116 * must take the slow path to
4117 * deal with the copy push
4118 */
4119 break;
4120 }
4121
4122 /*
4123 * This is now a shadow based copy on write
4124 * fault -- it requires a copy up the shadow
4125 * chain.
4126 */
4127 assert(m_object == VM_PAGE_OBJECT(m));
4128
4129 if ((cur_object_lock_type == OBJECT_LOCK_SHARED) &&
4130 VM_FAULT_NEED_CS_VALIDATION(NULL, m, m_object)) {
4131 goto upgrade_for_validation;
4132 }
4133
4134 /*
4135 * Allocate a page in the original top level
4136 * object. Give up if allocate fails. Also
4137 * need to remember current page, as it's the
4138 * source of the copy.
4139 *
4140 * at this point we hold locks on both
4141 * object and cur_object... no need to take
4142 * paging refs or mark pages BUSY since
4143 * we don't drop either object lock until
4144 * the page has been copied and inserted
4145 */
4146 cur_m = m;
4147 m = vm_page_grab_options(grab_options);
4148 m_object = NULL;
4149
4150 if (m == VM_PAGE_NULL) {
4151 /*
4152 * no free page currently available...
4153 * must take the slow path
4154 */
4155 break;
4156 }
4157 /*
4158 * Now do the copy. Mark the source page busy...
4159 *
4160 * NOTE: This code holds the map lock across
4161 * the page copy.
4162 */
4163 vm_page_copy(cur_m, m);
4164 vm_page_insert(m, object, offset);
4165 m_object = object;
4166 SET_PAGE_DIRTY(m, FALSE);
4167
4168 /*
4169 * Now cope with the source page and object
4170 */
4171 if (object->ref_count > 1 && cur_m->pmapped)
4172 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(cur_m));
4173
4174 if (cur_m->clustered) {
4175 VM_PAGE_COUNT_AS_PAGEIN(cur_m);
4176 VM_PAGE_CONSUME_CLUSTERED(cur_m);
4177 vm_fault_is_sequential(cur_object, cur_offset, fault_info.behavior);
4178 }
4179 need_collapse = TRUE;
4180
4181 if (!cur_object->internal &&
4182 cur_object->copy_strategy == MEMORY_OBJECT_COPY_DELAY) {
4183 /*
4184 * The object from which we've just
4185 * copied a page is most probably backed
4186 * by a vnode. We don't want to waste too
4187 * much time trying to collapse the VM objects
4188 * and create a bottleneck when several tasks
4189 * map the same file.
4190 */
4191 if (cur_object->copy == object) {
4192 /*
4193 * Shared mapping or no COW yet.
4194 * We can never collapse a copy
4195 * object into its backing object.
4196 */
4197 need_collapse = FALSE;
4198 } else if (cur_object->copy == object->shadow &&
4199 object->shadow->resident_page_count == 0) {
4200 /*
4201 * Shared mapping after a COW occurred.
4202 */
4203 need_collapse = FALSE;
4204 }
4205 }
4206 vm_object_unlock(cur_object);
4207
4208 if (need_collapse == FALSE)
4209 vm_fault_collapse_skipped++;
4210 vm_fault_collapse_total++;
4211
4212 type_of_fault = DBG_COW_FAULT;
4213 VM_STAT_INCR(cow_faults);
4214 DTRACE_VM2(cow_fault, int, 1, (uint64_t *), NULL);
4215 current_task()->cow_faults++;
4216
4217 goto FastPmapEnter;
4218
4219 } else {
4220 /*
4221 * No page at cur_object, cur_offset... m == NULL
4222 */
4223 if (cur_object->pager_created) {
4224 int compressor_external_state = VM_EXTERNAL_STATE_UNKNOWN;
4225
4226 if (MUST_ASK_PAGER(cur_object, cur_offset, compressor_external_state) == TRUE) {
4227 int my_fault_type;
4228 int c_flags = C_DONT_BLOCK;
4229 boolean_t insert_cur_object = FALSE;
4230
4231 /*
4232 * May have to talk to a pager...
4233 * if so, take the slow path by
4234 * doing a 'break' from the while (TRUE) loop
4235 *
4236 * external_state will only be set to VM_EXTERNAL_STATE_EXISTS
4237 * if the compressor is active and the page exists there
4238 */
4239 if (compressor_external_state != VM_EXTERNAL_STATE_EXISTS)
4240 break;
4241
4242 if (map == kernel_map || real_map == kernel_map) {
4243 /*
4244 * can't call into the compressor with the kernel_map
4245 * lock held, since the compressor may try to operate
4246 * on the kernel map in order to return an empty c_segment
4247 */
4248 break;
4249 }
4250 if (object != cur_object) {
4251 if (fault_type & VM_PROT_WRITE)
4252 c_flags |= C_KEEP;
4253 else
4254 insert_cur_object = TRUE;
4255 }
4256 if (insert_cur_object == TRUE) {
4257
4258 if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
4259
4260 cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4261
4262 if (vm_object_lock_upgrade(cur_object) == FALSE) {
4263 /*
4264 * couldn't upgrade so go do a full retry
4265 * immediately since we can no longer be
4266 * certain about cur_object (since we
4267 * don't hold a reference on it)...
4268 * first drop the top object lock
4269 */
4270 vm_object_unlock(object);
4271
4272 vm_map_unlock_read(map);
4273 if (real_map != map)
4274 vm_map_unlock(real_map);
4275
4276 goto RetryFault;
4277 }
4278 }
4279 } else if (object_lock_type == OBJECT_LOCK_SHARED) {
4280
4281 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4282
4283 if (object != cur_object) {
4284 /*
4285 * we can't go for the upgrade on the top
4286 * lock since the upgrade may block waiting
4287 * for readers to drain... since we hold
4288 * cur_object locked at this point, waiting
4289 * for the readers to drain would represent
4290 * a lock order inversion since the lock order
4291 * for objects is the reference order in the
4292 * shadown chain
4293 */
4294 vm_object_unlock(object);
4295 vm_object_unlock(cur_object);
4296
4297 vm_map_unlock_read(map);
4298 if (real_map != map)
4299 vm_map_unlock(real_map);
4300
4301 goto RetryFault;
4302 }
4303 if (vm_object_lock_upgrade(object) == FALSE) {
4304 /*
4305 * couldn't upgrade, so explictly take the lock
4306 * exclusively and go relookup the page since we
4307 * will have dropped the object lock and
4308 * a different thread could have inserted
4309 * a page at this offset
4310 * no need for a full retry since we're
4311 * at the top level of the object chain
4312 */
4313 vm_object_lock(object);
4314
4315 continue;
4316 }
4317 }
4318 m = vm_page_grab_options(grab_options);
4319 m_object = NULL;
4320
4321 if (m == VM_PAGE_NULL) {
4322 /*
4323 * no free page currently available...
4324 * must take the slow path
4325 */
4326 break;
4327 }
4328
4329 /*
4330 * The object is and remains locked
4331 * so no need to take a
4332 * "paging_in_progress" reference.
4333 */
4334 boolean_t shared_lock;
4335 if ((object == cur_object &&
4336 object_lock_type == OBJECT_LOCK_EXCLUSIVE) ||
4337 (object != cur_object &&
4338 cur_object_lock_type == OBJECT_LOCK_EXCLUSIVE)) {
4339 shared_lock = FALSE;
4340 } else {
4341 shared_lock = TRUE;
4342 }
4343
4344 kr = vm_compressor_pager_get(
4345 cur_object->pager,
4346 (cur_offset +
4347 cur_object->paging_offset),
4348 VM_PAGE_GET_PHYS_PAGE(m),
4349 &my_fault_type,
4350 c_flags,
4351 &compressed_count_delta);
4352
4353 vm_compressor_pager_count(
4354 cur_object->pager,
4355 compressed_count_delta,
4356 shared_lock,
4357 cur_object);
4358
4359 if (kr != KERN_SUCCESS) {
4360 vm_page_release(m, FALSE);
4361 m = VM_PAGE_NULL;
4362 break;
4363 }
4364 m->dirty = TRUE;
4365
4366 /*
4367 * If the object is purgeable, its
4368 * owner's purgeable ledgers will be
4369 * updated in vm_page_insert() but the
4370 * page was also accounted for in a
4371 * "compressed purgeable" ledger, so
4372 * update that now.
4373 */
4374 if (object != cur_object &&
4375 !insert_cur_object) {
4376 /*
4377 * We're not going to insert
4378 * the decompressed page into
4379 * the object it came from.
4380 *
4381 * We're dealing with a
4382 * copy-on-write fault on
4383 * "object".
4384 * We're going to decompress
4385 * the page directly into the
4386 * target "object" while
4387 * keepin the compressed
4388 * page for "cur_object", so
4389 * no ledger update in that
4390 * case.
4391 */
4392 } else if ((cur_object->purgable ==
4393 VM_PURGABLE_DENY) ||
4394 (cur_object->vo_purgeable_owner ==
4395 NULL)) {
4396 /*
4397 * "cur_object" is not purgeable
4398 * or is not owned, so no
4399 * purgeable ledgers to update.
4400 */
4401 } else {
4402 /*
4403 * One less compressed
4404 * purgeable page for
4405 * cur_object's owner.
4406 */
4407 vm_purgeable_compressed_update(
4408 cur_object,
4409 -1);
4410 }
4411
4412 if (insert_cur_object) {
4413 vm_page_insert(m, cur_object, cur_offset);
4414 m_object = cur_object;
4415 } else {
4416 vm_page_insert(m, object, offset);
4417 m_object = object;
4418 }
4419
4420 if ((m_object->wimg_bits & VM_WIMG_MASK) != VM_WIMG_USE_DEFAULT) {
4421 /*
4422 * If the page is not cacheable,
4423 * we can't let its contents
4424 * linger in the data cache
4425 * after the decompression.
4426 */
4427 pmap_sync_page_attributes_phys(VM_PAGE_GET_PHYS_PAGE(m));
4428 }
4429
4430 type_of_fault = my_fault_type;
4431
4432 VM_STAT_INCR(decompressions);
4433
4434 if (cur_object != object) {
4435 if (insert_cur_object) {
4436 top_object = object;
4437 /*
4438 * switch to the object that has the new page
4439 */
4440 object = cur_object;
4441 object_lock_type = cur_object_lock_type;
4442 } else {
4443 vm_object_unlock(cur_object);
4444 cur_object = object;
4445 }
4446 }
4447 goto FastPmapEnter;
4448 }
4449 /*
4450 * existence map present and indicates
4451 * that the pager doesn't have this page
4452 */
4453 }
4454 if (cur_object->shadow == VM_OBJECT_NULL) {
4455 /*
4456 * Zero fill fault. Page gets
4457 * inserted into the original object.
4458 */
4459 if (cur_object->shadow_severed ||
4460 VM_OBJECT_PURGEABLE_FAULT_ERROR(cur_object) ||
4461 cur_object == compressor_object ||
4462 cur_object == kernel_object ||
4463 cur_object == vm_submap_object) {
4464 if (object != cur_object)
4465 vm_object_unlock(cur_object);
4466 vm_object_unlock(object);
4467
4468 vm_map_unlock_read(map);
4469 if (real_map != map)
4470 vm_map_unlock(real_map);
4471
4472 kr = KERN_MEMORY_ERROR;
4473 goto done;
4474 }
4475 if (vm_backing_store_low) {
4476 /*
4477 * we are protecting the system from
4478 * backing store exhaustion...
4479 * must take the slow path if we're
4480 * not privileged
4481 */
4482 if (!(current_task()->priv_flags & VM_BACKING_STORE_PRIV))
4483 break;
4484 }
4485 if (cur_object != object) {
4486 vm_object_unlock(cur_object);
4487
4488 cur_object = object;
4489 }
4490 if (object_lock_type == OBJECT_LOCK_SHARED) {
4491
4492 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4493
4494 if (vm_object_lock_upgrade(object) == FALSE) {
4495 /*
4496 * couldn't upgrade so do a full retry on the fault
4497 * since we dropped the object lock which
4498 * could allow another thread to insert
4499 * a page at this offset
4500 */
4501 vm_map_unlock_read(map);
4502 if (real_map != map)
4503 vm_map_unlock(real_map);
4504
4505 goto RetryFault;
4506 }
4507 }
4508 m = vm_page_alloc(object, offset);
4509 m_object = NULL;
4510
4511 if (m == VM_PAGE_NULL) {
4512 /*
4513 * no free page currently available...
4514 * must take the slow path
4515 */
4516 break;
4517 }
4518 m_object = object;
4519
4520 /*
4521 * Now zero fill page...
4522 * the page is probably going to
4523 * be written soon, so don't bother
4524 * to clear the modified bit
4525 *
4526 * NOTE: This code holds the map
4527 * lock across the zero fill.
4528 */
4529 type_of_fault = vm_fault_zero_page(m, map->no_zero_fill);
4530
4531 goto FastPmapEnter;
4532 }
4533 /*
4534 * On to the next level in the shadow chain
4535 */
4536 cur_offset += cur_object->vo_shadow_offset;
4537 new_object = cur_object->shadow;
4538
4539 /*
4540 * take the new_object's lock with the indicated state
4541 */
4542 if (cur_object_lock_type == OBJECT_LOCK_SHARED)
4543 vm_object_lock_shared(new_object);
4544 else
4545 vm_object_lock(new_object);
4546
4547 if (cur_object != object)
4548 vm_object_unlock(cur_object);
4549
4550 cur_object = new_object;
4551
4552 continue;
4553 }
4554 }
4555 /*
4556 * Cleanup from fast fault failure. Drop any object
4557 * lock other than original and drop map lock.
4558 */
4559 if (object != cur_object)
4560 vm_object_unlock(cur_object);
4561
4562 /*
4563 * must own the object lock exclusively at this point
4564 */
4565 if (object_lock_type == OBJECT_LOCK_SHARED) {
4566 object_lock_type = OBJECT_LOCK_EXCLUSIVE;
4567
4568 if (vm_object_lock_upgrade(object) == FALSE) {
4569 /*
4570 * couldn't upgrade, so explictly
4571 * take the lock exclusively
4572 * no need to retry the fault at this
4573 * point since "vm_fault_page" will
4574 * completely re-evaluate the state
4575 */
4576 vm_object_lock(object);
4577 }
4578 }
4579
4580 handle_copy_delay:
4581 vm_map_unlock_read(map);
4582 if (real_map != map)
4583 vm_map_unlock(real_map);
4584
4585 assert(object != compressor_object);
4586 assert(object != kernel_object);
4587 assert(object != vm_submap_object);
4588
4589 /*
4590 * Make a reference to this object to
4591 * prevent its disposal while we are messing with
4592 * it. Once we have the reference, the map is free
4593 * to be diddled. Since objects reference their
4594 * shadows (and copies), they will stay around as well.
4595 */
4596 vm_object_reference_locked(object);
4597 vm_object_paging_begin(object);
4598
4599 XPR(XPR_VM_FAULT,"vm_fault -> vm_fault_page\n",0,0,0,0,0);
4600
4601 error_code = 0;
4602
4603 result_page = VM_PAGE_NULL;
4604 kr = vm_fault_page(object, offset, fault_type,
4605 (change_wiring && !wired),
4606 FALSE, /* page not looked up */
4607 &prot, &result_page, &top_page,
4608 &type_of_fault,
4609 &error_code, map->no_zero_fill,
4610 FALSE, &fault_info);
4611
4612 /*
4613 * if kr != VM_FAULT_SUCCESS, then the paging reference
4614 * has been dropped and the object unlocked... the ref_count
4615 * is still held
4616 *
4617 * if kr == VM_FAULT_SUCCESS, then the paging reference
4618 * is still held along with the ref_count on the original object
4619 *
4620 * the object is returned locked with a paging reference
4621 *
4622 * if top_page != NULL, then it's BUSY and the
4623 * object it belongs to has a paging reference
4624 * but is returned unlocked
4625 */
4626 if (kr != VM_FAULT_SUCCESS &&
4627 kr != VM_FAULT_SUCCESS_NO_VM_PAGE) {
4628 /*
4629 * we didn't succeed, lose the object reference immediately.
4630 */
4631 vm_object_deallocate(object);
4632
4633 /*
4634 * See why we failed, and take corrective action.
4635 */
4636 switch (kr) {
4637 case VM_FAULT_MEMORY_SHORTAGE:
4638 if (vm_page_wait((change_wiring) ?
4639 THREAD_UNINT :
4640 THREAD_ABORTSAFE))
4641 goto RetryFault;
4642 /*
4643 * fall thru
4644 */
4645 case VM_FAULT_INTERRUPTED:
4646 kr = KERN_ABORTED;
4647 goto done;
4648 case VM_FAULT_RETRY:
4649 goto RetryFault;
4650 case VM_FAULT_MEMORY_ERROR:
4651 if (error_code)
4652 kr = error_code;
4653 else
4654 kr = KERN_MEMORY_ERROR;
4655 goto done;
4656 default:
4657 panic("vm_fault: unexpected error 0x%x from "
4658 "vm_fault_page()\n", kr);
4659 }
4660 }
4661 m = result_page;
4662 m_object = NULL;
4663
4664 if (m != VM_PAGE_NULL) {
4665 m_object = VM_PAGE_OBJECT(m);
4666 assert((change_wiring && !wired) ?
4667 (top_page == VM_PAGE_NULL) :
4668 ((top_page == VM_PAGE_NULL) == (m_object == object)));
4669 }
4670
4671 /*
4672 * What to do with the resulting page from vm_fault_page
4673 * if it doesn't get entered into the physical map:
4674 */
4675 #define RELEASE_PAGE(m) \
4676 MACRO_BEGIN \
4677 PAGE_WAKEUP_DONE(m); \
4678 if ( !VM_PAGE_PAGEABLE(m)) { \
4679 vm_page_lockspin_queues(); \
4680 if ( !VM_PAGE_PAGEABLE(m)) \
4681 vm_page_activate(m); \
4682 vm_page_unlock_queues(); \
4683 } \
4684 MACRO_END
4685
4686 /*
4687 * We must verify that the maps have not changed
4688 * since our last lookup.
4689 */
4690 if (m != VM_PAGE_NULL) {
4691 old_copy_object = m_object->copy;
4692 vm_object_unlock(m_object);
4693 } else {
4694 old_copy_object = VM_OBJECT_NULL;
4695 vm_object_unlock(object);
4696 }
4697
4698 /*
4699 * no object locks are held at this point
4700 */
4701 if ((map != original_map) || !vm_map_verify(map, &version)) {
4702 vm_object_t retry_object;
4703 vm_object_offset_t retry_offset;
4704 vm_prot_t retry_prot;
4705
4706 /*
4707 * To avoid trying to write_lock the map while another
4708 * thread has it read_locked (in vm_map_pageable), we
4709 * do not try for write permission. If the page is
4710 * still writable, we will get write permission. If it
4711 * is not, or has been marked needs_copy, we enter the
4712 * mapping without write permission, and will merely
4713 * take another fault.
4714 */
4715 map = original_map;
4716 vm_map_lock_read(map);
4717
4718 kr = vm_map_lookup_locked(&map, vaddr,
4719 fault_type & ~VM_PROT_WRITE,
4720 OBJECT_LOCK_EXCLUSIVE, &version,
4721 &retry_object, &retry_offset, &retry_prot,
4722 &wired,
4723 &fault_info,
4724 &real_map);
4725 pmap = real_map->pmap;
4726
4727 if (kr != KERN_SUCCESS) {
4728 vm_map_unlock_read(map);
4729
4730 if (m != VM_PAGE_NULL) {
4731 assert(VM_PAGE_OBJECT(m) == m_object);
4732
4733 /*
4734 * retake the lock so that
4735 * we can drop the paging reference
4736 * in vm_fault_cleanup and do the
4737 * PAGE_WAKEUP_DONE in RELEASE_PAGE
4738 */
4739 vm_object_lock(m_object);
4740
4741 RELEASE_PAGE(m);
4742
4743 vm_fault_cleanup(m_object, top_page);
4744 } else {
4745 /*
4746 * retake the lock so that
4747 * we can drop the paging reference
4748 * in vm_fault_cleanup
4749 */
4750 vm_object_lock(object);
4751
4752 vm_fault_cleanup(object, top_page);
4753 }
4754 vm_object_deallocate(object);
4755
4756 goto done;
4757 }
4758 vm_object_unlock(retry_object);
4759
4760 if ((retry_object != object) || (retry_offset != offset)) {
4761
4762 vm_map_unlock_read(map);
4763 if (real_map != map)
4764 vm_map_unlock(real_map);
4765
4766 if (m != VM_PAGE_NULL) {
4767 assert(VM_PAGE_OBJECT(m) == m_object);
4768
4769 /*
4770 * retake the lock so that
4771 * we can drop the paging reference
4772 * in vm_fault_cleanup and do the
4773 * PAGE_WAKEUP_DONE in RELEASE_PAGE
4774 */
4775 vm_object_lock(m_object);
4776
4777 RELEASE_PAGE(m);
4778
4779 vm_fault_cleanup(m_object, top_page);
4780 } else {
4781 /*
4782 * retake the lock so that
4783 * we can drop the paging reference
4784 * in vm_fault_cleanup
4785 */
4786 vm_object_lock(object);
4787
4788 vm_fault_cleanup(object, top_page);
4789 }
4790 vm_object_deallocate(object);
4791
4792 goto RetryFault;
4793 }
4794 /*
4795 * Check whether the protection has changed or the object
4796 * has been copied while we left the map unlocked.
4797 */
4798 prot &= retry_prot;
4799 }
4800 if (m != VM_PAGE_NULL) {
4801 vm_object_lock(m_object);
4802
4803 if (m_object->copy != old_copy_object) {
4804 /*
4805 * The copy object changed while the top-level object
4806 * was unlocked, so take away write permission.
4807 */
4808 prot &= ~VM_PROT_WRITE;
4809 }
4810 } else
4811 vm_object_lock(object);
4812
4813 /*
4814 * If we want to wire down this page, but no longer have
4815 * adequate permissions, we must start all over.
4816 */
4817 if (wired && (fault_type != (prot | VM_PROT_WRITE))) {
4818
4819 vm_map_verify_done(map, &version);
4820 if (real_map != map)
4821 vm_map_unlock(real_map);
4822
4823 if (m != VM_PAGE_NULL) {
4824 assert(VM_PAGE_OBJECT(m) == m_object);
4825
4826 RELEASE_PAGE(m);
4827
4828 vm_fault_cleanup(m_object, top_page);
4829 } else
4830 vm_fault_cleanup(object, top_page);
4831
4832 vm_object_deallocate(object);
4833
4834 goto RetryFault;
4835 }
4836 if (m != VM_PAGE_NULL) {
4837 /*
4838 * Put this page into the physical map.
4839 * We had to do the unlock above because pmap_enter
4840 * may cause other faults. The page may be on
4841 * the pageout queues. If the pageout daemon comes
4842 * across the page, it will remove it from the queues.
4843 */
4844 if (caller_pmap) {
4845 kr = vm_fault_enter(m,
4846 caller_pmap,
4847 caller_pmap_addr,
4848 prot,
4849 caller_prot,
4850 wired,
4851 change_wiring,
4852 fault_info.no_cache,
4853 fault_info.cs_bypass,
4854 fault_info.user_tag,
4855 fault_info.pmap_options,
4856 NULL,
4857 &type_of_fault);
4858 } else {
4859 kr = vm_fault_enter(m,
4860 pmap,
4861 vaddr,
4862 prot,
4863 caller_prot,
4864 wired,
4865 change_wiring,
4866 fault_info.no_cache,
4867 fault_info.cs_bypass,
4868 fault_info.user_tag,
4869 fault_info.pmap_options,
4870 NULL,
4871 &type_of_fault);
4872 }
4873 assert(VM_PAGE_OBJECT(m) == m_object);
4874
4875 #if DEVELOPMENT || DEBUG
4876 {
4877 int event_code = 0;
4878
4879 if (m_object->internal)
4880 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_INTERNAL));
4881 else if (m_object->object_slid)
4882 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_SHAREDCACHE));
4883 else
4884 event_code = (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_REAL_FAULT_ADDR_EXTERNAL));
4885
4886 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, event_code, trace_real_vaddr, (fault_info.user_tag << 16) | (caller_prot << 8) | type_of_fault, m->offset, get_current_unique_pid(), 0);
4887
4888 DTRACE_VM6(real_fault, vm_map_offset_t, real_vaddr, vm_map_offset_t, m->offset, int, event_code, int, caller_prot, int, type_of_fault, int, fault_info.user_tag);
4889 }
4890 #endif
4891 if (kr != KERN_SUCCESS) {
4892 /* abort this page fault */
4893 vm_map_verify_done(map, &version);
4894 if (real_map != map)
4895 vm_map_unlock(real_map);
4896 PAGE_WAKEUP_DONE(m);
4897 vm_fault_cleanup(m_object, top_page);
4898 vm_object_deallocate(object);
4899 goto done;
4900 }
4901 if (physpage_p != NULL) {
4902 /* for vm_map_wire_and_extract() */
4903 *physpage_p = VM_PAGE_GET_PHYS_PAGE(m);
4904 if (prot & VM_PROT_WRITE) {
4905 vm_object_lock_assert_exclusive(m_object);
4906 m->dirty = TRUE;
4907 }
4908 }
4909 } else {
4910
4911 vm_map_entry_t entry;
4912 vm_map_offset_t laddr;
4913 vm_map_offset_t ldelta, hdelta;
4914
4915 /*
4916 * do a pmap block mapping from the physical address
4917 * in the object
4918 */
4919
4920 #ifdef ppc
4921 /* While we do not worry about execution protection in */
4922 /* general, certian pages may have instruction execution */
4923 /* disallowed. We will check here, and if not allowed */
4924 /* to execute, we return with a protection failure. */
4925
4926 if ((fault_type & VM_PROT_EXECUTE) &&
4927 (!pmap_eligible_for_execute((ppnum_t)(object->vo_shadow_offset >> 12)))) {
4928
4929 vm_map_verify_done(map, &version);
4930
4931 if (real_map != map)
4932 vm_map_unlock(real_map);
4933
4934 vm_fault_cleanup(object, top_page);
4935 vm_object_deallocate(object);
4936
4937 kr = KERN_PROTECTION_FAILURE;
4938 goto done;
4939 }
4940 #endif /* ppc */
4941
4942 if (real_map != map)
4943 vm_map_unlock(real_map);
4944
4945 if (original_map != map) {
4946 vm_map_unlock_read(map);
4947 vm_map_lock_read(original_map);
4948 map = original_map;
4949 }
4950 real_map = map;
4951
4952 laddr = vaddr;
4953 hdelta = 0xFFFFF000;
4954 ldelta = 0xFFFFF000;
4955
4956 while (vm_map_lookup_entry(map, laddr, &entry)) {
4957 if (ldelta > (laddr - entry->vme_start))
4958 ldelta = laddr - entry->vme_start;
4959 if (hdelta > (entry->vme_end - laddr))
4960 hdelta = entry->vme_end - laddr;
4961 if (entry->is_sub_map) {
4962
4963 laddr = ((laddr - entry->vme_start)
4964 + VME_OFFSET(entry));
4965 vm_map_lock_read(VME_SUBMAP(entry));
4966
4967 if (map != real_map)
4968 vm_map_unlock_read(map);
4969 if (entry->use_pmap) {
4970 vm_map_unlock_read(real_map);
4971 real_map = VME_SUBMAP(entry);
4972 }
4973 map = VME_SUBMAP(entry);
4974
4975 } else {
4976 break;
4977 }
4978 }
4979
4980 if (vm_map_lookup_entry(map, laddr, &entry) &&
4981 (VME_OBJECT(entry) != NULL) &&
4982 (VME_OBJECT(entry) == object)) {
4983 int superpage;
4984
4985 if (!object->pager_created &&
4986 object->phys_contiguous &&
4987 VME_OFFSET(entry) == 0 &&
4988 (entry->vme_end - entry->vme_start == object->vo_size) &&
4989 VM_MAP_PAGE_ALIGNED(entry->vme_start, (object->vo_size-1))) {
4990 superpage = VM_MEM_SUPERPAGE;
4991 } else {
4992 superpage = 0;
4993 }
4994
4995 if (superpage && physpage_p) {
4996 /* for vm_map_wire_and_extract() */
4997 *physpage_p = (ppnum_t)
4998 ((((vm_map_offset_t)
4999 object->vo_shadow_offset)
5000 + VME_OFFSET(entry)
5001 + (laddr - entry->vme_start))
5002 >> PAGE_SHIFT);
5003 }
5004
5005 if (caller_pmap) {
5006 /*
5007 * Set up a block mapped area
5008 */
5009 assert((uint32_t)((ldelta + hdelta) >> PAGE_SHIFT) == ((ldelta + hdelta) >> PAGE_SHIFT));
5010 pmap_map_block(caller_pmap,
5011 (addr64_t)(caller_pmap_addr - ldelta),
5012 (ppnum_t)((((vm_map_offset_t) (VME_OBJECT(entry)->vo_shadow_offset)) +
5013 VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT),
5014 (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot,
5015 (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0);
5016 } else {
5017 /*
5018 * Set up a block mapped area
5019 */
5020 assert((uint32_t)((ldelta + hdelta) >> PAGE_SHIFT) == ((ldelta + hdelta) >> PAGE_SHIFT));
5021 pmap_map_block(real_map->pmap,
5022 (addr64_t)(vaddr - ldelta),
5023 (ppnum_t)((((vm_map_offset_t)(VME_OBJECT(entry)->vo_shadow_offset)) +
5024 VME_OFFSET(entry) + (laddr - entry->vme_start) - ldelta) >> PAGE_SHIFT),
5025 (uint32_t)((ldelta + hdelta) >> PAGE_SHIFT), prot,
5026 (VM_WIMG_MASK & (int)object->wimg_bits) | superpage, 0);
5027 }
5028 }
5029 }
5030
5031 /*
5032 * Unlock everything, and return
5033 */
5034 vm_map_verify_done(map, &version);
5035 if (real_map != map)
5036 vm_map_unlock(real_map);
5037
5038 if (m != VM_PAGE_NULL) {
5039 assert(VM_PAGE_OBJECT(m) == m_object);
5040
5041 PAGE_WAKEUP_DONE(m);
5042
5043 vm_fault_cleanup(m_object, top_page);
5044 } else
5045 vm_fault_cleanup(object, top_page);
5046
5047 vm_object_deallocate(object);
5048
5049 #undef RELEASE_PAGE
5050
5051 kr = KERN_SUCCESS;
5052 done:
5053 thread_interrupt_level(interruptible_state);
5054
5055 /*
5056 * Only I/O throttle on faults which cause a pagein/swapin.
5057 */
5058 if ((type_of_fault == DBG_PAGEIND_FAULT) || (type_of_fault == DBG_PAGEINV_FAULT) || (type_of_fault == DBG_COMPRESSOR_SWAPIN_FAULT)) {
5059 throttle_lowpri_io(1);
5060 } else {
5061 if (kr == KERN_SUCCESS && type_of_fault != DBG_CACHE_HIT_FAULT && type_of_fault != DBG_GUARD_FAULT) {
5062
5063 if ((throttle_delay = vm_page_throttled(TRUE))) {
5064
5065 if (vm_debug_events) {
5066 if (type_of_fault == DBG_COMPRESSOR_FAULT)
5067 VM_DEBUG_EVENT(vmf_compressordelay, VMF_COMPRESSORDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
5068 else if (type_of_fault == DBG_COW_FAULT)
5069 VM_DEBUG_EVENT(vmf_cowdelay, VMF_COWDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
5070 else
5071 VM_DEBUG_EVENT(vmf_zfdelay, VMF_ZFDELAY, DBG_FUNC_NONE, throttle_delay, 0, 0, 0);
5072 }
5073 delay(throttle_delay);
5074 }
5075 }
5076 }
5077 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
5078 (MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END,
5079 ((uint64_t)trace_vaddr >> 32),
5080 trace_vaddr,
5081 kr,
5082 type_of_fault,
5083 0);
5084
5085 return (kr);
5086 }
5087
5088 /*
5089 * vm_fault_wire:
5090 *
5091 * Wire down a range of virtual addresses in a map.
5092 */
5093 kern_return_t
5094 vm_fault_wire(
5095 vm_map_t map,
5096 vm_map_entry_t entry,
5097 vm_prot_t prot,
5098 pmap_t pmap,
5099 vm_map_offset_t pmap_addr,
5100 ppnum_t *physpage_p)
5101 {
5102 vm_map_offset_t va;
5103 vm_map_offset_t end_addr = entry->vme_end;
5104 kern_return_t rc;
5105
5106 assert(entry->in_transition);
5107
5108 if ((VME_OBJECT(entry) != NULL) &&
5109 !entry->is_sub_map &&
5110 VME_OBJECT(entry)->phys_contiguous) {
5111 return KERN_SUCCESS;
5112 }
5113
5114 /*
5115 * Inform the physical mapping system that the
5116 * range of addresses may not fault, so that
5117 * page tables and such can be locked down as well.
5118 */
5119
5120 pmap_pageable(pmap, pmap_addr,
5121 pmap_addr + (end_addr - entry->vme_start), FALSE);
5122
5123 /*
5124 * We simulate a fault to get the page and enter it
5125 * in the physical map.
5126 */
5127
5128 for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
5129 rc = vm_fault_wire_fast(map, va, prot, entry, pmap,
5130 pmap_addr + (va - entry->vme_start),
5131 physpage_p);
5132 if (rc != KERN_SUCCESS) {
5133 rc = vm_fault_internal(map, va, prot, TRUE,
5134 ((pmap == kernel_pmap)
5135 ? THREAD_UNINT
5136 : THREAD_ABORTSAFE),
5137 pmap,
5138 (pmap_addr +
5139 (va - entry->vme_start)),
5140 physpage_p);
5141 DTRACE_VM2(softlock, int, 1, (uint64_t *), NULL);
5142 }
5143
5144 if (rc != KERN_SUCCESS) {
5145 struct vm_map_entry tmp_entry = *entry;
5146
5147 /* unwire wired pages */
5148 tmp_entry.vme_end = va;
5149 vm_fault_unwire(map,
5150 &tmp_entry, FALSE, pmap, pmap_addr);
5151
5152 return rc;
5153 }
5154 }
5155 return KERN_SUCCESS;
5156 }
5157
5158 /*
5159 * vm_fault_unwire:
5160 *
5161 * Unwire a range of virtual addresses in a map.
5162 */
5163 void
5164 vm_fault_unwire(
5165 vm_map_t map,
5166 vm_map_entry_t entry,
5167 boolean_t deallocate,
5168 pmap_t pmap,
5169 vm_map_offset_t pmap_addr)
5170 {
5171 vm_map_offset_t va;
5172 vm_map_offset_t end_addr = entry->vme_end;
5173 vm_object_t object;
5174 struct vm_object_fault_info fault_info;
5175
5176 object = (entry->is_sub_map) ? VM_OBJECT_NULL : VME_OBJECT(entry);
5177
5178 /*
5179 * If it's marked phys_contiguous, then vm_fault_wire() didn't actually
5180 * do anything since such memory is wired by default. So we don't have
5181 * anything to undo here.
5182 */
5183
5184 if (object != VM_OBJECT_NULL && object->phys_contiguous)
5185 return;
5186
5187 fault_info.interruptible = THREAD_UNINT;
5188 fault_info.behavior = entry->behavior;
5189 fault_info.user_tag = VME_ALIAS(entry);
5190 fault_info.pmap_options = 0;
5191 if (entry->iokit_acct ||
5192 (!entry->is_sub_map && !entry->use_pmap)) {
5193 fault_info.pmap_options |= PMAP_OPTIONS_ALT_ACCT;
5194 }
5195 fault_info.lo_offset = VME_OFFSET(entry);
5196 fault_info.hi_offset = (entry->vme_end - entry->vme_start) + VME_OFFSET(entry);
5197 fault_info.no_cache = entry->no_cache;
5198 fault_info.stealth = TRUE;
5199 fault_info.io_sync = FALSE;
5200 fault_info.cs_bypass = FALSE;
5201 fault_info.mark_zf_absent = FALSE;
5202 fault_info.batch_pmap_op = FALSE;
5203
5204 /*
5205 * Since the pages are wired down, we must be able to
5206 * get their mappings from the physical map system.
5207 */
5208
5209 for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
5210
5211 if (object == VM_OBJECT_NULL) {
5212 if (pmap) {
5213 pmap_change_wiring(pmap,
5214 pmap_addr + (va - entry->vme_start), FALSE);
5215 }
5216 (void) vm_fault(map, va, VM_PROT_NONE,
5217 TRUE, THREAD_UNINT, pmap, pmap_addr);
5218 } else {
5219 vm_prot_t prot;
5220 vm_page_t result_page;
5221 vm_page_t top_page;
5222 vm_object_t result_object;
5223 vm_fault_return_t result;
5224
5225 if (end_addr - va > (vm_size_t) -1) {
5226 /* 32-bit overflow */
5227 fault_info.cluster_size = (vm_size_t) (0 - PAGE_SIZE);
5228 } else {
5229 fault_info.cluster_size = (vm_size_t) (end_addr - va);
5230 assert(fault_info.cluster_size == end_addr - va);
5231 }
5232
5233 do {
5234 prot = VM_PROT_NONE;
5235
5236 vm_object_lock(object);
5237 vm_object_paging_begin(object);
5238 XPR(XPR_VM_FAULT,
5239 "vm_fault_unwire -> vm_fault_page\n",
5240 0,0,0,0,0);
5241 result_page = VM_PAGE_NULL;
5242 result = vm_fault_page(
5243 object,
5244 (VME_OFFSET(entry) +
5245 (va - entry->vme_start)),
5246 VM_PROT_NONE, TRUE,
5247 FALSE, /* page not looked up */
5248 &prot, &result_page, &top_page,
5249 (int *)0,
5250 NULL, map->no_zero_fill,
5251 FALSE, &fault_info);
5252 } while (result == VM_FAULT_RETRY);
5253
5254 /*
5255 * If this was a mapping to a file on a device that has been forcibly
5256 * unmounted, then we won't get a page back from vm_fault_page(). Just
5257 * move on to the next one in case the remaining pages are mapped from
5258 * different objects. During a forced unmount, the object is terminated
5259 * so the alive flag will be false if this happens. A forced unmount will
5260 * will occur when an external disk is unplugged before the user does an
5261 * eject, so we don't want to panic in that situation.
5262 */
5263
5264 if (result == VM_FAULT_MEMORY_ERROR && !object->alive)
5265 continue;
5266
5267 if (result == VM_FAULT_MEMORY_ERROR &&
5268 object == kernel_object) {
5269 /*
5270 * This must have been allocated with
5271 * KMA_KOBJECT and KMA_VAONLY and there's
5272 * no physical page at this offset.
5273 * We're done (no page to free).
5274 */
5275 assert(deallocate);
5276 continue;
5277 }
5278
5279 if (result != VM_FAULT_SUCCESS)
5280 panic("vm_fault_unwire: failure");
5281
5282 result_object = VM_PAGE_OBJECT(result_page);
5283
5284 if (deallocate) {
5285 assert(VM_PAGE_GET_PHYS_PAGE(result_page) !=
5286 vm_page_fictitious_addr);
5287 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(result_page));
5288 VM_PAGE_FREE(result_page);
5289 } else {
5290 if ((pmap) && (VM_PAGE_GET_PHYS_PAGE(result_page) != vm_page_guard_addr))
5291 pmap_change_wiring(pmap,
5292 pmap_addr + (va - entry->vme_start), FALSE);
5293
5294
5295 if (VM_PAGE_WIRED(result_page)) {
5296 vm_page_lockspin_queues();
5297 vm_page_unwire(result_page, TRUE);
5298 vm_page_unlock_queues();
5299 }
5300 if(entry->zero_wired_pages) {
5301 pmap_zero_page(VM_PAGE_GET_PHYS_PAGE(result_page));
5302 entry->zero_wired_pages = FALSE;
5303 }
5304
5305 PAGE_WAKEUP_DONE(result_page);
5306 }
5307 vm_fault_cleanup(result_object, top_page);
5308 }
5309 }
5310
5311 /*
5312 * Inform the physical mapping system that the range
5313 * of addresses may fault, so that page tables and
5314 * such may be unwired themselves.
5315 */
5316
5317 pmap_pageable(pmap, pmap_addr,
5318 pmap_addr + (end_addr - entry->vme_start), TRUE);
5319
5320 }
5321
5322 /*
5323 * vm_fault_wire_fast:
5324 *
5325 * Handle common case of a wire down page fault at the given address.
5326 * If successful, the page is inserted into the associated physical map.
5327 * The map entry is passed in to avoid the overhead of a map lookup.
5328 *
5329 * NOTE: the given address should be truncated to the
5330 * proper page address.
5331 *
5332 * KERN_SUCCESS is returned if the page fault is handled; otherwise,
5333 * a standard error specifying why the fault is fatal is returned.
5334 *
5335 * The map in question must be referenced, and remains so.
5336 * Caller has a read lock on the map.
5337 *
5338 * This is a stripped version of vm_fault() for wiring pages. Anything
5339 * other than the common case will return KERN_FAILURE, and the caller
5340 * is expected to call vm_fault().
5341 */
5342 static kern_return_t
5343 vm_fault_wire_fast(
5344 __unused vm_map_t map,
5345 vm_map_offset_t va,
5346 vm_prot_t caller_prot,
5347 vm_map_entry_t entry,
5348 pmap_t pmap,
5349 vm_map_offset_t pmap_addr,
5350 ppnum_t *physpage_p)
5351 {
5352 vm_object_t object;
5353 vm_object_offset_t offset;
5354 vm_page_t m;
5355 vm_prot_t prot;
5356 thread_t thread = current_thread();
5357 int type_of_fault;
5358 kern_return_t kr;
5359
5360 VM_STAT_INCR(faults);
5361
5362 if (thread != THREAD_NULL && thread->task != TASK_NULL)
5363 thread->task->faults++;
5364
5365 /*
5366 * Recovery actions
5367 */
5368
5369 #undef RELEASE_PAGE
5370 #define RELEASE_PAGE(m) { \
5371 PAGE_WAKEUP_DONE(m); \
5372 vm_page_lockspin_queues(); \
5373 vm_page_unwire(m, TRUE); \
5374 vm_page_unlock_queues(); \
5375 }
5376
5377
5378 #undef UNLOCK_THINGS
5379 #define UNLOCK_THINGS { \
5380 vm_object_paging_end(object); \
5381 vm_object_unlock(object); \
5382 }
5383
5384 #undef UNLOCK_AND_DEALLOCATE
5385 #define UNLOCK_AND_DEALLOCATE { \
5386 UNLOCK_THINGS; \
5387 vm_object_deallocate(object); \
5388 }
5389 /*
5390 * Give up and have caller do things the hard way.
5391 */
5392
5393 #define GIVE_UP { \
5394 UNLOCK_AND_DEALLOCATE; \
5395 return(KERN_FAILURE); \
5396 }
5397
5398
5399 /*
5400 * If this entry is not directly to a vm_object, bail out.
5401 */
5402 if (entry->is_sub_map) {
5403 assert(physpage_p == NULL);
5404 return(KERN_FAILURE);
5405 }
5406
5407 /*
5408 * Find the backing store object and offset into it.
5409 */
5410
5411 object = VME_OBJECT(entry);
5412 offset = (va - entry->vme_start) + VME_OFFSET(entry);
5413 prot = entry->protection;
5414
5415 /*
5416 * Make a reference to this object to prevent its
5417 * disposal while we are messing with it.
5418 */
5419
5420 vm_object_lock(object);
5421 vm_object_reference_locked(object);
5422 vm_object_paging_begin(object);
5423
5424 /*
5425 * INVARIANTS (through entire routine):
5426 *
5427 * 1) At all times, we must either have the object
5428 * lock or a busy page in some object to prevent
5429 * some other thread from trying to bring in
5430 * the same page.
5431 *
5432 * 2) Once we have a busy page, we must remove it from
5433 * the pageout queues, so that the pageout daemon
5434 * will not grab it away.
5435 *
5436 */
5437
5438 /*
5439 * Look for page in top-level object. If it's not there or
5440 * there's something going on, give up.
5441 * ENCRYPTED SWAP: use the slow fault path, since we'll need to
5442 * decrypt the page before wiring it down.
5443 */
5444 m = vm_page_lookup(object, offset);
5445 if ((m == VM_PAGE_NULL) || (m->busy) || (m->encrypted) ||
5446 (m->unusual && ( m->error || m->restart || m->absent))) {
5447
5448 GIVE_UP;
5449 }
5450 ASSERT_PAGE_DECRYPTED(m);
5451
5452 if (m->fictitious &&
5453 VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
5454 /*
5455 * Guard pages are fictitious pages and are never
5456 * entered into a pmap, so let's say it's been wired...
5457 */
5458 kr = KERN_SUCCESS;
5459 goto done;
5460 }
5461
5462 /*
5463 * Wire the page down now. All bail outs beyond this
5464 * point must unwire the page.
5465 */
5466
5467 vm_page_lockspin_queues();
5468 vm_page_wire(m, VM_PROT_MEMORY_TAG(caller_prot), TRUE);
5469 vm_page_unlock_queues();
5470
5471 /*
5472 * Mark page busy for other threads.
5473 */
5474 assert(!m->busy);
5475 m->busy = TRUE;
5476 assert(!m->absent);
5477
5478 /*
5479 * Give up if the page is being written and there's a copy object
5480 */
5481 if ((object->copy != VM_OBJECT_NULL) && (prot & VM_PROT_WRITE)) {
5482 RELEASE_PAGE(m);
5483 GIVE_UP;
5484 }
5485
5486 /*
5487 * Put this page into the physical map.
5488 */
5489 type_of_fault = DBG_CACHE_HIT_FAULT;
5490 kr = vm_fault_enter(m,
5491 pmap,
5492 pmap_addr,
5493 prot,
5494 prot,
5495 TRUE,
5496 FALSE,
5497 FALSE,
5498 FALSE,
5499 VME_ALIAS(entry),
5500 ((entry->iokit_acct ||
5501 (!entry->is_sub_map && !entry->use_pmap))
5502 ? PMAP_OPTIONS_ALT_ACCT
5503 : 0),
5504 NULL,
5505 &type_of_fault);
5506 if (kr != KERN_SUCCESS) {
5507 RELEASE_PAGE(m);
5508 GIVE_UP;
5509 }
5510
5511 done:
5512 /*
5513 * Unlock everything, and return
5514 */
5515
5516 if (physpage_p) {
5517 /* for vm_map_wire_and_extract() */
5518 if (kr == KERN_SUCCESS) {
5519 assert(object == VM_PAGE_OBJECT(m));
5520 *physpage_p = VM_PAGE_GET_PHYS_PAGE(m);
5521 if (prot & VM_PROT_WRITE) {
5522 vm_object_lock_assert_exclusive(object);
5523 m->dirty = TRUE;
5524 }
5525 } else {
5526 *physpage_p = 0;
5527 }
5528 }
5529
5530 PAGE_WAKEUP_DONE(m);
5531 UNLOCK_AND_DEALLOCATE;
5532
5533 return kr;
5534
5535 }
5536
5537 /*
5538 * Routine: vm_fault_copy_cleanup
5539 * Purpose:
5540 * Release a page used by vm_fault_copy.
5541 */
5542
5543 static void
5544 vm_fault_copy_cleanup(
5545 vm_page_t page,
5546 vm_page_t top_page)
5547 {
5548 vm_object_t object = VM_PAGE_OBJECT(page);
5549
5550 vm_object_lock(object);
5551 PAGE_WAKEUP_DONE(page);
5552 if ( !VM_PAGE_PAGEABLE(page)) {
5553 vm_page_lockspin_queues();
5554 if ( !VM_PAGE_PAGEABLE(page)) {
5555 vm_page_activate(page);
5556 }
5557 vm_page_unlock_queues();
5558 }
5559 vm_fault_cleanup(object, top_page);
5560 }
5561
5562 static void
5563 vm_fault_copy_dst_cleanup(
5564 vm_page_t page)
5565 {
5566 vm_object_t object;
5567
5568 if (page != VM_PAGE_NULL) {
5569 object = VM_PAGE_OBJECT(page);
5570 vm_object_lock(object);
5571 vm_page_lockspin_queues();
5572 vm_page_unwire(page, TRUE);
5573 vm_page_unlock_queues();
5574 vm_object_paging_end(object);
5575 vm_object_unlock(object);
5576 }
5577 }
5578
5579 /*
5580 * Routine: vm_fault_copy
5581 *
5582 * Purpose:
5583 * Copy pages from one virtual memory object to another --
5584 * neither the source nor destination pages need be resident.
5585 *
5586 * Before actually copying a page, the version associated with
5587 * the destination address map wil be verified.
5588 *
5589 * In/out conditions:
5590 * The caller must hold a reference, but not a lock, to
5591 * each of the source and destination objects and to the
5592 * destination map.
5593 *
5594 * Results:
5595 * Returns KERN_SUCCESS if no errors were encountered in
5596 * reading or writing the data. Returns KERN_INTERRUPTED if
5597 * the operation was interrupted (only possible if the
5598 * "interruptible" argument is asserted). Other return values
5599 * indicate a permanent error in copying the data.
5600 *
5601 * The actual amount of data copied will be returned in the
5602 * "copy_size" argument. In the event that the destination map
5603 * verification failed, this amount may be less than the amount
5604 * requested.
5605 */
5606 kern_return_t
5607 vm_fault_copy(
5608 vm_object_t src_object,
5609 vm_object_offset_t src_offset,
5610 vm_map_size_t *copy_size, /* INOUT */
5611 vm_object_t dst_object,
5612 vm_object_offset_t dst_offset,
5613 vm_map_t dst_map,
5614 vm_map_version_t *dst_version,
5615 int interruptible)
5616 {
5617 vm_page_t result_page;
5618
5619 vm_page_t src_page;
5620 vm_page_t src_top_page;
5621 vm_prot_t src_prot;
5622
5623 vm_page_t dst_page;
5624 vm_page_t dst_top_page;
5625 vm_prot_t dst_prot;
5626
5627 vm_map_size_t amount_left;
5628 vm_object_t old_copy_object;
5629 vm_object_t result_page_object = NULL;
5630 kern_return_t error = 0;
5631 vm_fault_return_t result;
5632
5633 vm_map_size_t part_size;
5634 struct vm_object_fault_info fault_info_src;
5635 struct vm_object_fault_info fault_info_dst;
5636
5637 /*
5638 * In order not to confuse the clustered pageins, align
5639 * the different offsets on a page boundary.
5640 */
5641
5642 #define RETURN(x) \
5643 MACRO_BEGIN \
5644 *copy_size -= amount_left; \
5645 MACRO_RETURN(x); \
5646 MACRO_END
5647
5648 amount_left = *copy_size;
5649
5650 fault_info_src.interruptible = interruptible;
5651 fault_info_src.behavior = VM_BEHAVIOR_SEQUENTIAL;
5652 fault_info_src.user_tag = 0;
5653 fault_info_src.pmap_options = 0;
5654 fault_info_src.lo_offset = vm_object_trunc_page(src_offset);
5655 fault_info_src.hi_offset = fault_info_src.lo_offset + amount_left;
5656 fault_info_src.no_cache = FALSE;
5657 fault_info_src.stealth = TRUE;
5658 fault_info_src.io_sync = FALSE;
5659 fault_info_src.cs_bypass = FALSE;
5660 fault_info_src.mark_zf_absent = FALSE;
5661 fault_info_src.batch_pmap_op = FALSE;
5662
5663 fault_info_dst.interruptible = interruptible;
5664 fault_info_dst.behavior = VM_BEHAVIOR_SEQUENTIAL;
5665 fault_info_dst.user_tag = 0;
5666 fault_info_dst.pmap_options = 0;
5667 fault_info_dst.lo_offset = vm_object_trunc_page(dst_offset);
5668 fault_info_dst.hi_offset = fault_info_dst.lo_offset + amount_left;
5669 fault_info_dst.no_cache = FALSE;
5670 fault_info_dst.stealth = TRUE;
5671 fault_info_dst.io_sync = FALSE;
5672 fault_info_dst.cs_bypass = FALSE;
5673 fault_info_dst.mark_zf_absent = FALSE;
5674 fault_info_dst.batch_pmap_op = FALSE;
5675
5676 do { /* while (amount_left > 0) */
5677 /*
5678 * There may be a deadlock if both source and destination
5679 * pages are the same. To avoid this deadlock, the copy must
5680 * start by getting the destination page in order to apply
5681 * COW semantics if any.
5682 */
5683
5684 RetryDestinationFault: ;
5685
5686 dst_prot = VM_PROT_WRITE|VM_PROT_READ;
5687
5688 vm_object_lock(dst_object);
5689 vm_object_paging_begin(dst_object);
5690
5691 if (amount_left > (vm_size_t) -1) {
5692 /* 32-bit overflow */
5693 fault_info_dst.cluster_size = (vm_size_t) (0 - PAGE_SIZE);
5694 } else {
5695 fault_info_dst.cluster_size = (vm_size_t) amount_left;
5696 assert(fault_info_dst.cluster_size == amount_left);
5697 }
5698
5699 XPR(XPR_VM_FAULT,"vm_fault_copy -> vm_fault_page\n",0,0,0,0,0);
5700 dst_page = VM_PAGE_NULL;
5701 result = vm_fault_page(dst_object,
5702 vm_object_trunc_page(dst_offset),
5703 VM_PROT_WRITE|VM_PROT_READ,
5704 FALSE,
5705 FALSE, /* page not looked up */
5706 &dst_prot, &dst_page, &dst_top_page,
5707 (int *)0,
5708 &error,
5709 dst_map->no_zero_fill,
5710 FALSE, &fault_info_dst);
5711 switch (result) {
5712 case VM_FAULT_SUCCESS:
5713 break;
5714 case VM_FAULT_RETRY:
5715 goto RetryDestinationFault;
5716 case VM_FAULT_MEMORY_SHORTAGE:
5717 if (vm_page_wait(interruptible))
5718 goto RetryDestinationFault;
5719 /* fall thru */
5720 case VM_FAULT_INTERRUPTED:
5721 RETURN(MACH_SEND_INTERRUPTED);
5722 case VM_FAULT_SUCCESS_NO_VM_PAGE:
5723 /* success but no VM page: fail the copy */
5724 vm_object_paging_end(dst_object);
5725 vm_object_unlock(dst_object);
5726 /*FALLTHROUGH*/
5727 case VM_FAULT_MEMORY_ERROR:
5728 if (error)
5729 return (error);
5730 else
5731 return(KERN_MEMORY_ERROR);
5732 default:
5733 panic("vm_fault_copy: unexpected error 0x%x from "
5734 "vm_fault_page()\n", result);
5735 }
5736 assert ((dst_prot & VM_PROT_WRITE) != VM_PROT_NONE);
5737
5738 assert(dst_object == VM_PAGE_OBJECT(dst_page));
5739 old_copy_object = dst_object->copy;
5740
5741 /*
5742 * There exists the possiblity that the source and
5743 * destination page are the same. But we can't
5744 * easily determine that now. If they are the
5745 * same, the call to vm_fault_page() for the
5746 * destination page will deadlock. To prevent this we
5747 * wire the page so we can drop busy without having
5748 * the page daemon steal the page. We clean up the
5749 * top page but keep the paging reference on the object
5750 * holding the dest page so it doesn't go away.
5751 */
5752
5753 vm_page_lockspin_queues();
5754 vm_page_wire(dst_page, VM_KERN_MEMORY_OSFMK, TRUE);
5755 vm_page_unlock_queues();
5756 PAGE_WAKEUP_DONE(dst_page);
5757 vm_object_unlock(dst_object);
5758
5759 if (dst_top_page != VM_PAGE_NULL) {
5760 vm_object_lock(dst_object);
5761 VM_PAGE_FREE(dst_top_page);
5762 vm_object_paging_end(dst_object);
5763 vm_object_unlock(dst_object);
5764 }
5765
5766 RetrySourceFault: ;
5767
5768 if (src_object == VM_OBJECT_NULL) {
5769 /*
5770 * No source object. We will just
5771 * zero-fill the page in dst_object.
5772 */
5773 src_page = VM_PAGE_NULL;
5774 result_page = VM_PAGE_NULL;
5775 } else {
5776 vm_object_lock(src_object);
5777 src_page = vm_page_lookup(src_object,
5778 vm_object_trunc_page(src_offset));
5779 if (src_page == dst_page) {
5780 src_prot = dst_prot;
5781 result_page = VM_PAGE_NULL;
5782 } else {
5783 src_prot = VM_PROT_READ;
5784 vm_object_paging_begin(src_object);
5785
5786 if (amount_left > (vm_size_t) -1) {
5787 /* 32-bit overflow */
5788 fault_info_src.cluster_size = (vm_size_t) (0 - PAGE_SIZE);
5789 } else {
5790 fault_info_src.cluster_size = (vm_size_t) amount_left;
5791 assert(fault_info_src.cluster_size == amount_left);
5792 }
5793
5794 XPR(XPR_VM_FAULT,
5795 "vm_fault_copy(2) -> vm_fault_page\n",
5796 0,0,0,0,0);
5797 result_page = VM_PAGE_NULL;
5798 result = vm_fault_page(
5799 src_object,
5800 vm_object_trunc_page(src_offset),
5801 VM_PROT_READ, FALSE,
5802 FALSE, /* page not looked up */
5803 &src_prot,
5804 &result_page, &src_top_page,
5805 (int *)0, &error, FALSE,
5806 FALSE, &fault_info_src);
5807
5808 switch (result) {
5809 case VM_FAULT_SUCCESS:
5810 break;
5811 case VM_FAULT_RETRY:
5812 goto RetrySourceFault;
5813 case VM_FAULT_MEMORY_SHORTAGE:
5814 if (vm_page_wait(interruptible))
5815 goto RetrySourceFault;
5816 /* fall thru */
5817 case VM_FAULT_INTERRUPTED:
5818 vm_fault_copy_dst_cleanup(dst_page);
5819 RETURN(MACH_SEND_INTERRUPTED);
5820 case VM_FAULT_SUCCESS_NO_VM_PAGE:
5821 /* success but no VM page: fail */
5822 vm_object_paging_end(src_object);
5823 vm_object_unlock(src_object);
5824 /*FALLTHROUGH*/
5825 case VM_FAULT_MEMORY_ERROR:
5826 vm_fault_copy_dst_cleanup(dst_page);
5827 if (error)
5828 return (error);
5829 else
5830 return(KERN_MEMORY_ERROR);
5831 default:
5832 panic("vm_fault_copy(2): unexpected "
5833 "error 0x%x from "
5834 "vm_fault_page()\n", result);
5835 }
5836
5837 result_page_object = VM_PAGE_OBJECT(result_page);
5838 assert((src_top_page == VM_PAGE_NULL) ==
5839 (result_page_object == src_object));
5840 }
5841 assert ((src_prot & VM_PROT_READ) != VM_PROT_NONE);
5842 vm_object_unlock(result_page_object);
5843 }
5844
5845 if (!vm_map_verify(dst_map, dst_version)) {
5846 if (result_page != VM_PAGE_NULL && src_page != dst_page)
5847 vm_fault_copy_cleanup(result_page, src_top_page);
5848 vm_fault_copy_dst_cleanup(dst_page);
5849 break;
5850 }
5851 assert(dst_object == VM_PAGE_OBJECT(dst_page));
5852
5853 vm_object_lock(dst_object);
5854
5855 if (dst_object->copy != old_copy_object) {
5856 vm_object_unlock(dst_object);
5857 vm_map_verify_done(dst_map, dst_version);
5858 if (result_page != VM_PAGE_NULL && src_page != dst_page)
5859 vm_fault_copy_cleanup(result_page, src_top_page);
5860 vm_fault_copy_dst_cleanup(dst_page);
5861 break;
5862 }
5863 vm_object_unlock(dst_object);
5864
5865 /*
5866 * Copy the page, and note that it is dirty
5867 * immediately.
5868 */
5869
5870 if (!page_aligned(src_offset) ||
5871 !page_aligned(dst_offset) ||
5872 !page_aligned(amount_left)) {
5873
5874 vm_object_offset_t src_po,
5875 dst_po;
5876
5877 src_po = src_offset - vm_object_trunc_page(src_offset);
5878 dst_po = dst_offset - vm_object_trunc_page(dst_offset);
5879
5880 if (dst_po > src_po) {
5881 part_size = PAGE_SIZE - dst_po;
5882 } else {
5883 part_size = PAGE_SIZE - src_po;
5884 }
5885 if (part_size > (amount_left)){
5886 part_size = amount_left;
5887 }
5888
5889 if (result_page == VM_PAGE_NULL) {
5890 assert((vm_offset_t) dst_po == dst_po);
5891 assert((vm_size_t) part_size == part_size);
5892 vm_page_part_zero_fill(dst_page,
5893 (vm_offset_t) dst_po,
5894 (vm_size_t) part_size);
5895 } else {
5896 assert((vm_offset_t) src_po == src_po);
5897 assert((vm_offset_t) dst_po == dst_po);
5898 assert((vm_size_t) part_size == part_size);
5899 vm_page_part_copy(result_page,
5900 (vm_offset_t) src_po,
5901 dst_page,
5902 (vm_offset_t) dst_po,
5903 (vm_size_t)part_size);
5904 if(!dst_page->dirty){
5905 vm_object_lock(dst_object);
5906 SET_PAGE_DIRTY(dst_page, TRUE);
5907 vm_object_unlock(dst_object);
5908 }
5909
5910 }
5911 } else {
5912 part_size = PAGE_SIZE;
5913
5914 if (result_page == VM_PAGE_NULL)
5915 vm_page_zero_fill(dst_page);
5916 else{
5917 vm_object_lock(result_page_object);
5918 vm_page_copy(result_page, dst_page);
5919 vm_object_unlock(result_page_object);
5920
5921 if(!dst_page->dirty){
5922 vm_object_lock(dst_object);
5923 SET_PAGE_DIRTY(dst_page, TRUE);
5924 vm_object_unlock(dst_object);
5925 }
5926 }
5927
5928 }
5929
5930 /*
5931 * Unlock everything, and return
5932 */
5933
5934 vm_map_verify_done(dst_map, dst_version);
5935
5936 if (result_page != VM_PAGE_NULL && src_page != dst_page)
5937 vm_fault_copy_cleanup(result_page, src_top_page);
5938 vm_fault_copy_dst_cleanup(dst_page);
5939
5940 amount_left -= part_size;
5941 src_offset += part_size;
5942 dst_offset += part_size;
5943 } while (amount_left > 0);
5944
5945 RETURN(KERN_SUCCESS);
5946 #undef RETURN
5947
5948 /*NOTREACHED*/
5949 }
5950
5951 #if VM_FAULT_CLASSIFY
5952 /*
5953 * Temporary statistics gathering support.
5954 */
5955
5956 /*
5957 * Statistics arrays:
5958 */
5959 #define VM_FAULT_TYPES_MAX 5
5960 #define VM_FAULT_LEVEL_MAX 8
5961
5962 int vm_fault_stats[VM_FAULT_TYPES_MAX][VM_FAULT_LEVEL_MAX];
5963
5964 #define VM_FAULT_TYPE_ZERO_FILL 0
5965 #define VM_FAULT_TYPE_MAP_IN 1
5966 #define VM_FAULT_TYPE_PAGER 2
5967 #define VM_FAULT_TYPE_COPY 3
5968 #define VM_FAULT_TYPE_OTHER 4
5969
5970
5971 void
5972 vm_fault_classify(vm_object_t object,
5973 vm_object_offset_t offset,
5974 vm_prot_t fault_type)
5975 {
5976 int type, level = 0;
5977 vm_page_t m;
5978
5979 while (TRUE) {
5980 m = vm_page_lookup(object, offset);
5981 if (m != VM_PAGE_NULL) {
5982 if (m->busy || m->error || m->restart || m->absent) {
5983 type = VM_FAULT_TYPE_OTHER;
5984 break;
5985 }
5986 if (((fault_type & VM_PROT_WRITE) == 0) ||
5987 ((level == 0) && object->copy == VM_OBJECT_NULL)) {
5988 type = VM_FAULT_TYPE_MAP_IN;
5989 break;
5990 }
5991 type = VM_FAULT_TYPE_COPY;
5992 break;
5993 }
5994 else {
5995 if (object->pager_created) {
5996 type = VM_FAULT_TYPE_PAGER;
5997 break;
5998 }
5999 if (object->shadow == VM_OBJECT_NULL) {
6000 type = VM_FAULT_TYPE_ZERO_FILL;
6001 break;
6002 }
6003
6004 offset += object->vo_shadow_offset;
6005 object = object->shadow;
6006 level++;
6007 continue;
6008 }
6009 }
6010
6011 if (level > VM_FAULT_LEVEL_MAX)
6012 level = VM_FAULT_LEVEL_MAX;
6013
6014 vm_fault_stats[type][level] += 1;
6015
6016 return;
6017 }
6018
6019 /* cleanup routine to call from debugger */
6020
6021 void
6022 vm_fault_classify_init(void)
6023 {
6024 int type, level;
6025
6026 for (type = 0; type < VM_FAULT_TYPES_MAX; type++) {
6027 for (level = 0; level < VM_FAULT_LEVEL_MAX; level++) {
6028 vm_fault_stats[type][level] = 0;
6029 }
6030 }
6031
6032 return;
6033 }
6034 #endif /* VM_FAULT_CLASSIFY */
6035
6036 vm_offset_t
6037 kdp_lightweight_fault(vm_map_t map, vm_offset_t cur_target_addr)
6038 {
6039 vm_map_entry_t entry;
6040 vm_object_t object;
6041 vm_offset_t object_offset;
6042 vm_page_t m;
6043 int compressor_external_state, compressed_count_delta;
6044 int compressor_flags = (C_DONT_BLOCK | C_KEEP | C_KDP);
6045 int my_fault_type = VM_PROT_READ;
6046 kern_return_t kr;
6047
6048 if (not_in_kdp) {
6049 panic("kdp_lightweight_fault called from outside of debugger context");
6050 }
6051
6052 assert(map != VM_MAP_NULL);
6053
6054 assert((cur_target_addr & PAGE_MASK) == 0);
6055 if ((cur_target_addr & PAGE_MASK) != 0) {
6056 return 0;
6057 }
6058
6059 if (kdp_lck_rw_lock_is_acquired_exclusive(&map->lock)) {
6060 return 0;
6061 }
6062
6063 if (!vm_map_lookup_entry(map, cur_target_addr, &entry)) {
6064 return 0;
6065 }
6066
6067 if (entry->is_sub_map) {
6068 return 0;
6069 }
6070
6071 object = VME_OBJECT(entry);
6072 if (object == VM_OBJECT_NULL) {
6073 return 0;
6074 }
6075
6076 object_offset = cur_target_addr - entry->vme_start + VME_OFFSET(entry);
6077
6078 while (TRUE) {
6079 if (kdp_lck_rw_lock_is_acquired_exclusive(&object->Lock)) {
6080 return 0;
6081 }
6082
6083 if (object->pager_created && (object->paging_in_progress ||
6084 object->activity_in_progress)) {
6085 return 0;
6086 }
6087
6088 m = kdp_vm_page_lookup(object, object_offset);
6089
6090 if (m != VM_PAGE_NULL) {
6091
6092 if ((object->wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
6093 return 0;
6094 }
6095
6096 if (m->laundry || m->busy || m->free_when_done || m->absent || m->error || m->cleaning ||
6097 m->overwriting || m->restart || m->unusual) {
6098 return 0;
6099 }
6100
6101 assert(!m->private);
6102 if (m->private) {
6103 return 0;
6104 }
6105
6106 assert(!m->fictitious);
6107 if (m->fictitious) {
6108 return 0;
6109 }
6110
6111 assert(!m->encrypted);
6112 if (m->encrypted) {
6113 return 0;
6114 }
6115
6116 assert(!m->encrypted_cleaning);
6117 if (m->encrypted_cleaning) {
6118 return 0;
6119 }
6120
6121 assert(m->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR);
6122 if (m->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
6123 return 0;
6124 }
6125
6126 return ptoa(VM_PAGE_GET_PHYS_PAGE(m));
6127 }
6128
6129 compressor_external_state = VM_EXTERNAL_STATE_UNKNOWN;
6130
6131 if (object->pager_created && MUST_ASK_PAGER(object, object_offset, compressor_external_state)) {
6132 if (compressor_external_state == VM_EXTERNAL_STATE_EXISTS) {
6133 kr = vm_compressor_pager_get(object->pager, (object_offset + object->paging_offset),
6134 kdp_compressor_decompressed_page_ppnum, &my_fault_type,
6135 compressor_flags, &compressed_count_delta);
6136 if (kr == KERN_SUCCESS) {
6137 return kdp_compressor_decompressed_page_paddr;
6138 } else {
6139 return 0;
6140 }
6141 }
6142 }
6143
6144 if (object->shadow == VM_OBJECT_NULL) {
6145 return 0;
6146 }
6147
6148 object_offset += object->vo_shadow_offset;
6149 object = object->shadow;
6150 }
6151
6152 }
6153
6154 void
6155 vm_page_validate_cs_mapped(
6156 vm_page_t page,
6157 const void *kaddr)
6158 {
6159 vm_object_t object;
6160 vm_object_offset_t offset;
6161 memory_object_t pager;
6162 struct vnode *vnode;
6163 boolean_t validated;
6164 unsigned tainted;
6165
6166 assert(page->busy);
6167 object = VM_PAGE_OBJECT(page);
6168 vm_object_lock_assert_exclusive(object);
6169
6170 if (page->wpmapped && !page->cs_tainted) {
6171 /*
6172 * This page was mapped for "write" access sometime in the
6173 * past and could still be modifiable in the future.
6174 * Consider it tainted.
6175 * [ If the page was already found to be "tainted", no
6176 * need to re-validate. ]
6177 */
6178 page->cs_validated = TRUE;
6179 page->cs_tainted = TRUE;
6180 if (cs_debug) {
6181 printf("CODESIGNING: vm_page_validate_cs: "
6182 "page %p obj %p off 0x%llx "
6183 "was modified\n",
6184 page, object, page->offset);
6185 }
6186 vm_cs_validated_dirtied++;
6187 }
6188
6189 if (page->cs_validated || page->cs_tainted) {
6190 return;
6191 }
6192
6193 vm_cs_validates++;
6194
6195 assert(object->code_signed);
6196 offset = page->offset;
6197
6198 if (!object->alive || object->terminating || object->pager == NULL) {
6199 /*
6200 * The object is terminating and we don't have its pager
6201 * so we can't validate the data...
6202 */
6203 return;
6204 }
6205 /*
6206 * Since we get here to validate a page that was brought in by
6207 * the pager, we know that this pager is all setup and ready
6208 * by now.
6209 */
6210 assert(!object->internal);
6211 assert(object->pager != NULL);
6212 assert(object->pager_ready);
6213
6214 pager = object->pager;
6215 assert(object->paging_in_progress);
6216 vnode = vnode_pager_lookup_vnode(pager);
6217
6218 /* verify the SHA1 hash for this page */
6219 tainted = 0;
6220 validated = cs_validate_range(vnode,
6221 pager,
6222 (object->paging_offset +
6223 offset),
6224 (const void *)((const char *)kaddr),
6225 PAGE_SIZE_64,
6226 &tainted);
6227
6228 if (tainted & CS_VALIDATE_TAINTED) {
6229 page->cs_tainted = TRUE;
6230 }
6231 if (tainted & CS_VALIDATE_NX) {
6232 page->cs_nx = TRUE;
6233 }
6234
6235 if (validated) {
6236 page->cs_validated = TRUE;
6237 }
6238 }
6239
6240 void
6241 vm_page_validate_cs(
6242 vm_page_t page)
6243 {
6244 vm_object_t object;
6245 vm_object_offset_t offset;
6246 vm_map_offset_t koffset;
6247 vm_map_size_t ksize;
6248 vm_offset_t kaddr;
6249 kern_return_t kr;
6250 boolean_t busy_page;
6251 boolean_t need_unmap;
6252
6253 object = VM_PAGE_OBJECT(page);
6254 vm_object_lock_assert_held(object);
6255
6256 if (page->wpmapped && !page->cs_tainted) {
6257 vm_object_lock_assert_exclusive(object);
6258
6259 /*
6260 * This page was mapped for "write" access sometime in the
6261 * past and could still be modifiable in the future.
6262 * Consider it tainted.
6263 * [ If the page was already found to be "tainted", no
6264 * need to re-validate. ]
6265 */
6266 page->cs_validated = TRUE;
6267 page->cs_tainted = TRUE;
6268 if (cs_debug) {
6269 printf("CODESIGNING: vm_page_validate_cs: "
6270 "page %p obj %p off 0x%llx "
6271 "was modified\n",
6272 page, object, page->offset);
6273 }
6274 vm_cs_validated_dirtied++;
6275 }
6276
6277 if (page->cs_validated || page->cs_tainted) {
6278 return;
6279 }
6280
6281 if (page->slid) {
6282 panic("vm_page_validate_cs(%p): page is slid\n", page);
6283 }
6284 assert(!page->slid);
6285
6286 #if CHECK_CS_VALIDATION_BITMAP
6287 if ( vnode_pager_cs_check_validation_bitmap( object->pager, trunc_page(page->offset + object->paging_offset), CS_BITMAP_CHECK ) == KERN_SUCCESS) {
6288 page->cs_validated = TRUE;
6289 page->cs_tainted = FALSE;
6290 vm_cs_bitmap_validated++;
6291 return;
6292 }
6293 #endif
6294 vm_object_lock_assert_exclusive(object);
6295
6296 assert(object->code_signed);
6297 offset = page->offset;
6298
6299 busy_page = page->busy;
6300 if (!busy_page) {
6301 /* keep page busy while we map (and unlock) the VM object */
6302 page->busy = TRUE;
6303 }
6304
6305 /*
6306 * Take a paging reference on the VM object
6307 * to protect it from collapse or bypass,
6308 * and keep it from disappearing too.
6309 */
6310 vm_object_paging_begin(object);
6311
6312 /* map the page in the kernel address space */
6313 ksize = PAGE_SIZE_64;
6314 koffset = 0;
6315 need_unmap = FALSE;
6316 kr = vm_paging_map_object(page,
6317 object,
6318 offset,
6319 VM_PROT_READ,
6320 FALSE, /* can't unlock object ! */
6321 &ksize,
6322 &koffset,
6323 &need_unmap);
6324 if (kr != KERN_SUCCESS) {
6325 panic("vm_page_validate_cs: could not map page: 0x%x\n", kr);
6326 }
6327 kaddr = CAST_DOWN(vm_offset_t, koffset);
6328
6329 /* validate the mapped page */
6330 vm_page_validate_cs_mapped(page, (const void *) kaddr);
6331
6332 #if CHECK_CS_VALIDATION_BITMAP
6333 if ( page->cs_validated == TRUE && page->cs_tainted == FALSE ) {
6334 vnode_pager_cs_check_validation_bitmap( object->pager, trunc_page( offset + object->paging_offset), CS_BITMAP_SET );
6335 }
6336 #endif
6337 assert(page->busy);
6338 assert(object == VM_PAGE_OBJECT(page));
6339 vm_object_lock_assert_exclusive(object);
6340
6341 if (!busy_page) {
6342 PAGE_WAKEUP_DONE(page);
6343 }
6344 if (need_unmap) {
6345 /* unmap the map from the kernel address space */
6346 vm_paging_unmap_object(object, koffset, koffset + ksize);
6347 koffset = 0;
6348 ksize = 0;
6349 kaddr = 0;
6350 }
6351 vm_object_paging_end(object);
6352 }
6353
6354 void
6355 vm_page_validate_cs_mapped_chunk(
6356 vm_page_t page,
6357 const void *kaddr,
6358 vm_offset_t chunk_offset,
6359 vm_size_t chunk_size,
6360 boolean_t *validated_p,
6361 unsigned *tainted_p)
6362 {
6363 vm_object_t object;
6364 vm_object_offset_t offset, offset_in_page;
6365 memory_object_t pager;
6366 struct vnode *vnode;
6367 boolean_t validated;
6368 unsigned tainted;
6369
6370 *validated_p = FALSE;
6371 *tainted_p = 0;
6372
6373 assert(page->busy);
6374 object = VM_PAGE_OBJECT(page);
6375 vm_object_lock_assert_exclusive(object);
6376
6377 assert(object->code_signed);
6378 offset = page->offset;
6379
6380 if (!object->alive || object->terminating || object->pager == NULL) {
6381 /*
6382 * The object is terminating and we don't have its pager
6383 * so we can't validate the data...
6384 */
6385 return;
6386 }
6387 /*
6388 * Since we get here to validate a page that was brought in by
6389 * the pager, we know that this pager is all setup and ready
6390 * by now.
6391 */
6392 assert(!object->internal);
6393 assert(object->pager != NULL);
6394 assert(object->pager_ready);
6395
6396 pager = object->pager;
6397 assert(object->paging_in_progress);
6398 vnode = vnode_pager_lookup_vnode(pager);
6399
6400 /* verify the signature for this chunk */
6401 offset_in_page = chunk_offset;
6402 assert(offset_in_page < PAGE_SIZE);
6403
6404 tainted = 0;
6405 validated = cs_validate_range(vnode,
6406 pager,
6407 (object->paging_offset +
6408 offset +
6409 offset_in_page),
6410 (const void *)((const char *)kaddr
6411 + offset_in_page),
6412 chunk_size,
6413 &tainted);
6414 if (validated) {
6415 *validated_p = TRUE;
6416 }
6417 if (tainted) {
6418 *tainted_p = tainted;
6419 }
6420 }
mirror of XNU