]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/bsd_vm.c
projects / apple / xnu.git / blob
? search:


e4cad6a94a2bde78b4e59cabb4f6f4d645942c79
[apple/xnu.git] / osfmk / vm / bsd_vm.c
1 /*
2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the
10 * License may not be used to create, or enable the creation or
11 * redistribution of, unlawful or unlicensed copies of an Apple operating
12 * system, or to circumvent, violate, or enable the circumvention or
13 * violation of, any terms of an Apple operating system software license
14 * agreement.
15 *
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this
18 * file.
19 *
20 * The Original Code and all software distributed under the License are
21 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
22 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
23 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
25 * Please see the License for the specific language governing rights and
26 * limitations under the License.
27 *
28 * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
29 */
30
31 #include <sys/errno.h>
32
33 #include <mach/mach_types.h>
34 #include <mach/mach_traps.h>
35 #include <mach/host_priv.h>
36 #include <mach/kern_return.h>
37 #include <mach/memory_object_control.h>
38 #include <mach/memory_object_types.h>
39 #include <mach/port.h>
40 #include <mach/policy.h>
41 #include <mach/upl.h>
42 #include <mach/thread_act.h>
43
44 #include <kern/host.h>
45 #include <kern/thread.h>
46
47 #include <ipc/ipc_port.h>
48 #include <ipc/ipc_space.h>
49
50 #include <default_pager/default_pager_types.h>
51 #include <default_pager/default_pager_object_server.h>
52
53 #include <vm/vm_map.h>
54 #include <vm/vm_pageout.h>
55 #include <vm/memory_object.h>
56 #include <vm/vm_pageout.h>
57 #include <vm/vm_protos.h>
58
59 /* BSD VM COMPONENT INTERFACES */
60 int
61 get_map_nentries(
62 vm_map_t);
63
64 vm_offset_t
65 get_map_start(
66 vm_map_t);
67
68 vm_offset_t
69 get_map_end(
70 vm_map_t);
71
72 /*
73 *
74 */
75 int
76 get_map_nentries(
77 vm_map_t map)
78 {
79 return(map->hdr.nentries);
80 }
81
82 mach_vm_offset_t
83 mach_get_vm_start(vm_map_t map)
84 {
85 return( vm_map_first_entry(map)->vme_start);
86 }
87
88 mach_vm_offset_t
89 mach_get_vm_end(vm_map_t map)
90 {
91 return( vm_map_last_entry(map)->vme_end);
92 }
93
94 /*
95 * Legacy routines to get the start and end for a vm_map_t. They
96 * return them in the vm_offset_t format. So, they should only be
97 * called on maps that are the same size as the kernel map for
98 * accurate results.
99 */
100 vm_offset_t
101 get_vm_start(
102 vm_map_t map)
103 {
104 return(CAST_DOWN(vm_offset_t, vm_map_first_entry(map)->vme_start));
105 }
106
107 vm_offset_t
108 get_vm_end(
109 vm_map_t map)
110 {
111 return(CAST_DOWN(vm_offset_t, vm_map_last_entry(map)->vme_end));
112 }
113
114 /*
115 * BSD VNODE PAGER
116 */
117
118 const struct memory_object_pager_ops vnode_pager_ops = {
119 vnode_pager_reference,
120 vnode_pager_deallocate,
121 vnode_pager_init,
122 vnode_pager_terminate,
123 vnode_pager_data_request,
124 vnode_pager_data_return,
125 vnode_pager_data_initialize,
126 vnode_pager_data_unlock,
127 vnode_pager_synchronize,
128 vnode_pager_unmap,
129 "vnode pager"
130 };
131
132 typedef struct vnode_pager {
133 memory_object_pager_ops_t pager_ops; /* == &vnode_pager_ops */
134 unsigned int pager_ikot; /* JMM: fake ip_kotype() */
135 unsigned int ref_count; /* reference count */
136 memory_object_control_t control_handle; /* mem object control handle */
137 struct vnode *vnode_handle; /* vnode handle */
138 } *vnode_pager_t;
139
140
141 ipc_port_t
142 trigger_name_to_port( /* forward */
143 mach_port_t);
144
145 kern_return_t
146 vnode_pager_cluster_read( /* forward */
147 vnode_pager_t,
148 vm_object_offset_t,
149 vm_size_t);
150
151 void
152 vnode_pager_cluster_write( /* forward */
153 vnode_pager_t,
154 vm_object_offset_t,
155 vm_size_t,
156 vm_object_offset_t *,
157 int *,
158 int);
159
160
161 vnode_pager_t
162 vnode_object_create( /* forward */
163 struct vnode *);
164
165 vnode_pager_t
166 vnode_pager_lookup( /* forward */
167 memory_object_t);
168
169 zone_t vnode_pager_zone;
170
171
172 #define VNODE_PAGER_NULL ((vnode_pager_t) 0)
173
174 /* TODO: Should be set dynamically by vnode_pager_init() */
175 #define CLUSTER_SHIFT 1
176
177 /* TODO: Should be set dynamically by vnode_pager_bootstrap() */
178 #define MAX_VNODE 10000
179
180
181 #if DEBUG
182 int pagerdebug=0;
183
184 #define PAGER_ALL 0xffffffff
185 #define PAGER_INIT 0x00000001
186 #define PAGER_PAGEIN 0x00000002
187
188 #define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
189 #else
190 #define PAGER_DEBUG(LEVEL, A)
191 #endif
192
193 /*
194 * Routine: macx_triggers
195 * Function:
196 * Syscall interface to set the call backs for low and
197 * high water marks.
198 */
199 int
200 macx_triggers(
201 struct macx_triggers_args *args)
202 {
203 int hi_water = args->hi_water;
204 int low_water = args->low_water;
205 int flags = args->flags;
206 mach_port_t trigger_name = args->alert_port;
207 kern_return_t kr;
208 memory_object_default_t default_pager;
209 ipc_port_t trigger_port;
210
211 default_pager = MEMORY_OBJECT_DEFAULT_NULL;
212 kr = host_default_memory_manager(host_priv_self(),
213 &default_pager, 0);
214 if(kr != KERN_SUCCESS) {
215 return EINVAL;
216 }
217
218 if ((flags & SWAP_ENCRYPT_ON) &&
219 (flags & SWAP_ENCRYPT_OFF)) {
220 /* can't have it both ways */
221 return EINVAL;
222 }
223
224 if (default_pager_init_flag == 0) {
225 start_def_pager(NULL);
226 default_pager_init_flag = 1;
227 }
228
229 if (flags & SWAP_ENCRYPT_ON) {
230 /* ENCRYPTED SWAP: tell default_pager to encrypt */
231 default_pager_triggers(default_pager,
232 0, 0,
233 SWAP_ENCRYPT_ON,
234 IP_NULL);
235 } else if (flags & SWAP_ENCRYPT_OFF) {
236 /* ENCRYPTED SWAP: tell default_pager not to encrypt */
237 default_pager_triggers(default_pager,
238 0, 0,
239 SWAP_ENCRYPT_OFF,
240 IP_NULL);
241 }
242
243 if (flags & HI_WAT_ALERT) {
244 trigger_port = trigger_name_to_port(trigger_name);
245 if(trigger_port == NULL) {
246 return EINVAL;
247 }
248 /* trigger_port is locked and active */
249 ipc_port_make_send_locked(trigger_port);
250 /* now unlocked */
251 default_pager_triggers(default_pager,
252 hi_water, low_water,
253 HI_WAT_ALERT, trigger_port);
254 }
255
256 if (flags & LO_WAT_ALERT) {
257 trigger_port = trigger_name_to_port(trigger_name);
258 if(trigger_port == NULL) {
259 return EINVAL;
260 }
261 /* trigger_port is locked and active */
262 ipc_port_make_send_locked(trigger_port);
263 /* and now its unlocked */
264 default_pager_triggers(default_pager,
265 hi_water, low_water,
266 LO_WAT_ALERT, trigger_port);
267 }
268
269 /*
270 * Set thread scheduling priority and policy for the current thread
271 * it is assumed for the time being that the thread setting the alert
272 * is the same one which will be servicing it.
273 *
274 * XXX This does not belong in the kernel XXX
275 */
276 {
277 thread_precedence_policy_data_t pre;
278 thread_extended_policy_data_t ext;
279
280 ext.timeshare = FALSE;
281 pre.importance = INT32_MAX;
282
283 thread_policy_set(current_thread(),
284 THREAD_EXTENDED_POLICY,
285 (thread_policy_t)&ext,
286 THREAD_EXTENDED_POLICY_COUNT);
287
288 thread_policy_set(current_thread(),
289 THREAD_PRECEDENCE_POLICY,
290 (thread_policy_t)&pre,
291 THREAD_PRECEDENCE_POLICY_COUNT);
292 }
293
294 current_thread()->options |= TH_OPT_VMPRIV;
295
296 return 0;
297 }
298
299 /*
300 *
301 */
302 ipc_port_t
303 trigger_name_to_port(
304 mach_port_t trigger_name)
305 {
306 ipc_port_t trigger_port;
307 ipc_space_t space;
308
309 if (trigger_name == 0)
310 return (NULL);
311
312 space = current_space();
313 if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name,
314 &trigger_port) != KERN_SUCCESS)
315 return (NULL);
316 return trigger_port;
317 }
318
319
320 extern int uiomove64(addr64_t, int, void *);
321 #define MAX_RUN 32
322
323 int
324 memory_object_control_uiomove(
325 memory_object_control_t control,
326 memory_object_offset_t offset,
327 void * uio,
328 int start_offset,
329 int io_requested,
330 int mark_dirty)
331 {
332 vm_object_t object;
333 vm_page_t dst_page;
334 int xsize;
335 int retval = 0;
336 int cur_run;
337 int cur_needed;
338 int i;
339 vm_page_t page_run[MAX_RUN];
340
341
342 object = memory_object_control_to_vm_object(control);
343 if (object == VM_OBJECT_NULL) {
344 return (0);
345 }
346 assert(!object->internal);
347
348 vm_object_lock(object);
349
350 if (mark_dirty && object->copy != VM_OBJECT_NULL) {
351 /*
352 * We can't modify the pages without honoring
353 * copy-on-write obligations first, so fall off
354 * this optimized path and fall back to the regular
355 * path.
356 */
357 vm_object_unlock(object);
358 return 0;
359 }
360
361 while (io_requested && retval == 0) {
362
363 cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE;
364
365 if (cur_needed > MAX_RUN)
366 cur_needed = MAX_RUN;
367
368 for (cur_run = 0; cur_run < cur_needed; ) {
369
370 if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL)
371 break;
372 /*
373 * Sync up on getting the busy bit
374 */
375 if ((dst_page->busy || dst_page->cleaning)) {
376 /*
377 * someone else is playing with the page... if we've
378 * already collected pages into this run, go ahead
379 * and process now, we can't block on this
380 * page while holding other pages in the BUSY state
381 * otherwise we will wait
382 */
383 if (cur_run)
384 break;
385 PAGE_SLEEP(object, dst_page, THREAD_UNINT);
386 continue;
387 }
388 /*
389 * this routine is only called when copying
390 * to/from real files... no need to consider
391 * encrypted swap pages
392 */
393 assert(!dst_page->encrypted);
394
395 if (mark_dirty)
396 dst_page->dirty = TRUE;
397 dst_page->busy = TRUE;
398
399 page_run[cur_run++] = dst_page;
400
401 offset += PAGE_SIZE_64;
402 }
403 if (cur_run == 0)
404 /*
405 * we hit a 'hole' in the cache
406 * we bail at this point
407 * we'll unlock the object below
408 */
409 break;
410 vm_object_unlock(object);
411
412 for (i = 0; i < cur_run; i++) {
413
414 dst_page = page_run[i];
415
416 if ((xsize = PAGE_SIZE - start_offset) > io_requested)
417 xsize = io_requested;
418
419 if ( (retval = uiomove64((addr64_t)(((addr64_t)(dst_page->phys_page) << 12) + start_offset), xsize, uio)) )
420 break;
421
422 io_requested -= xsize;
423 start_offset = 0;
424 }
425 vm_object_lock(object);
426
427 for (i = 0; i < cur_run; i++) {
428 dst_page = page_run[i];
429
430 PAGE_WAKEUP_DONE(dst_page);
431 }
432 }
433 vm_object_unlock(object);
434
435 return (retval);
436 }
437
438
439 /*
440 *
441 */
442 void
443 vnode_pager_bootstrap(void)
444 {
445 register vm_size_t size;
446
447 size = (vm_size_t) sizeof(struct vnode_pager);
448 vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
449 PAGE_SIZE, "vnode pager structures");
450 #ifdef __i386__
451 apple_protect_pager_bootstrap();
452 #endif /* __i386__ */
453 return;
454 }
455
456 /*
457 *
458 */
459 memory_object_t
460 vnode_pager_setup(
461 struct vnode *vp,
462 __unused memory_object_t pager)
463 {
464 vnode_pager_t vnode_object;
465
466 vnode_object = vnode_object_create(vp);
467 if (vnode_object == VNODE_PAGER_NULL)
468 panic("vnode_pager_setup: vnode_object_create() failed");
469 return((memory_object_t)vnode_object);
470 }
471
472 /*
473 *
474 */
475 kern_return_t
476 vnode_pager_init(memory_object_t mem_obj,
477 memory_object_control_t control,
478 #if !DEBUG
479 __unused
480 #endif
481 vm_size_t pg_size)
482 {
483 vnode_pager_t vnode_object;
484 kern_return_t kr;
485 memory_object_attr_info_data_t attributes;
486
487
488 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %p, %p, %x\n", mem_obj, control, pg_size));
489
490 if (control == MEMORY_OBJECT_CONTROL_NULL)
491 return KERN_INVALID_ARGUMENT;
492
493 vnode_object = vnode_pager_lookup(mem_obj);
494
495 memory_object_control_reference(control);
496
497 vnode_object->control_handle = control;
498
499 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
500 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
501 attributes.cluster_size = (1 << (PAGE_SHIFT));
502 attributes.may_cache_object = TRUE;
503 attributes.temporary = TRUE;
504
505 kr = memory_object_change_attributes(
506 control,
507 MEMORY_OBJECT_ATTRIBUTE_INFO,
508 (memory_object_info_t) &attributes,
509 MEMORY_OBJECT_ATTR_INFO_COUNT);
510 if (kr != KERN_SUCCESS)
511 panic("vnode_pager_init: memory_object_change_attributes() failed");
512
513 return(KERN_SUCCESS);
514 }
515
516 /*
517 *
518 */
519 kern_return_t
520 vnode_pager_data_return(
521 memory_object_t mem_obj,
522 memory_object_offset_t offset,
523 vm_size_t data_cnt,
524 memory_object_offset_t *resid_offset,
525 int *io_error,
526 __unused boolean_t dirty,
527 __unused boolean_t kernel_copy,
528 int upl_flags)
529 {
530 register vnode_pager_t vnode_object;
531
532 vnode_object = vnode_pager_lookup(mem_obj);
533
534 vnode_pager_cluster_write(vnode_object, offset, data_cnt, resid_offset, io_error, upl_flags);
535
536 return KERN_SUCCESS;
537 }
538
539 kern_return_t
540 vnode_pager_data_initialize(
541 __unused memory_object_t mem_obj,
542 __unused memory_object_offset_t offset,
543 __unused vm_size_t data_cnt)
544 {
545 panic("vnode_pager_data_initialize");
546 return KERN_FAILURE;
547 }
548
549 kern_return_t
550 vnode_pager_data_unlock(
551 __unused memory_object_t mem_obj,
552 __unused memory_object_offset_t offset,
553 __unused vm_size_t size,
554 __unused vm_prot_t desired_access)
555 {
556 return KERN_FAILURE;
557 }
558
559 kern_return_t
560 vnode_pager_get_object_size(
561 memory_object_t mem_obj,
562 memory_object_offset_t *length)
563 {
564 vnode_pager_t vnode_object;
565
566 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
567 *length = 0;
568 return KERN_INVALID_ARGUMENT;
569 }
570
571 vnode_object = vnode_pager_lookup(mem_obj);
572
573 *length = vnode_pager_get_filesize(vnode_object->vnode_handle);
574 return KERN_SUCCESS;
575 }
576
577 kern_return_t
578 vnode_pager_get_object_pathname(
579 memory_object_t mem_obj,
580 char *pathname,
581 vm_size_t *length_p)
582 {
583 vnode_pager_t vnode_object;
584
585 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
586 return KERN_INVALID_ARGUMENT;
587 }
588
589 vnode_object = vnode_pager_lookup(mem_obj);
590
591 return vnode_pager_get_pathname(vnode_object->vnode_handle,
592 pathname,
593 length_p);
594 }
595
596 kern_return_t
597 vnode_pager_get_object_filename(
598 memory_object_t mem_obj,
599 char **filename)
600 {
601 vnode_pager_t vnode_object;
602
603 if (mem_obj->mo_pager_ops != &vnode_pager_ops) {
604 return KERN_INVALID_ARGUMENT;
605 }
606
607 vnode_object = vnode_pager_lookup(mem_obj);
608
609 return vnode_pager_get_filename(vnode_object->vnode_handle,
610 filename);
611 }
612
613 /*
614 *
615 */
616 kern_return_t
617 vnode_pager_data_request(
618 memory_object_t mem_obj,
619 memory_object_offset_t offset,
620 vm_size_t length,
621 #if !DEBUG
622 __unused
623 #endif
624 vm_prot_t protection_required)
625 {
626 register vnode_pager_t vnode_object;
627
628 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required));
629
630 vnode_object = vnode_pager_lookup(mem_obj);
631
632 PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object));
633
634 return vnode_pager_cluster_read(vnode_object, offset, length);
635 }
636
637 /*
638 *
639 */
640 void
641 vnode_pager_reference(
642 memory_object_t mem_obj)
643 {
644 register vnode_pager_t vnode_object;
645 unsigned int new_ref_count;
646
647 vnode_object = vnode_pager_lookup(mem_obj);
648 new_ref_count = hw_atomic_add(&vnode_object->ref_count, 1);
649 assert(new_ref_count > 1);
650 }
651
652 /*
653 *
654 */
655 void
656 vnode_pager_deallocate(
657 memory_object_t mem_obj)
658 {
659 register vnode_pager_t vnode_object;
660
661 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj));
662
663 vnode_object = vnode_pager_lookup(mem_obj);
664
665 if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) {
666 if (vnode_object->vnode_handle != NULL) {
667 vnode_pager_vrele(vnode_object->vnode_handle);
668 }
669 zfree(vnode_pager_zone, vnode_object);
670 }
671 return;
672 }
673
674 /*
675 *
676 */
677 kern_return_t
678 vnode_pager_terminate(
679 #if !DEBUG
680 __unused
681 #endif
682 memory_object_t mem_obj)
683 {
684 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj));
685
686 return(KERN_SUCCESS);
687 }
688
689 /*
690 *
691 */
692 kern_return_t
693 vnode_pager_synchronize(
694 memory_object_t mem_obj,
695 memory_object_offset_t offset,
696 vm_size_t length,
697 __unused vm_sync_t sync_flags)
698 {
699 register vnode_pager_t vnode_object;
700
701 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj));
702
703 vnode_object = vnode_pager_lookup(mem_obj);
704
705 memory_object_synchronize_completed(vnode_object->control_handle, offset, length);
706
707 return (KERN_SUCCESS);
708 }
709
710 /*
711 *
712 */
713 kern_return_t
714 vnode_pager_unmap(
715 memory_object_t mem_obj)
716 {
717 register vnode_pager_t vnode_object;
718
719 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj));
720
721 vnode_object = vnode_pager_lookup(mem_obj);
722
723 ubc_unmap(vnode_object->vnode_handle);
724 return KERN_SUCCESS;
725 }
726
727
728 /*
729 *
730 */
731 void
732 vnode_pager_cluster_write(
733 vnode_pager_t vnode_object,
734 vm_object_offset_t offset,
735 vm_size_t cnt,
736 vm_object_offset_t * resid_offset,
737 int * io_error,
738 int upl_flags)
739 {
740 vm_size_t size;
741 upl_t upl = NULL;
742 int request_flags;
743 int errno;
744
745 if (upl_flags & UPL_MSYNC) {
746
747 upl_flags |= UPL_VNODE_PAGER;
748
749 if ( (upl_flags & UPL_IOSYNC) && io_error)
750 upl_flags |= UPL_KEEPCACHED;
751
752 while (cnt) {
753 kern_return_t kr;
754
755 size = (cnt < (PAGE_SIZE * MAX_UPL_TRANSFER)) ? cnt : (PAGE_SIZE * MAX_UPL_TRANSFER); /* effective max */
756
757 request_flags = UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | UPL_CLEAN_IN_PLACE |
758 UPL_SET_INTERNAL | UPL_SET_LITE;
759
760 kr = memory_object_upl_request(vnode_object->control_handle,
761 offset, size, &upl, NULL, NULL, request_flags);
762 if (kr != KERN_SUCCESS)
763 panic("vnode_pager_cluster_write: upl request failed\n");
764
765 vnode_pageout(vnode_object->vnode_handle,
766 upl, (vm_offset_t)0, offset, size, upl_flags, &errno);
767
768 if ( (upl_flags & UPL_KEEPCACHED) ) {
769 if ( (*io_error = errno) )
770 break;
771 }
772 cnt -= size;
773 offset += size;
774 }
775 if (resid_offset)
776 *resid_offset = offset;
777
778 } else {
779 vm_object_offset_t vnode_size;
780 vm_object_offset_t base_offset;
781 vm_object_t object;
782 vm_page_t target_page;
783 int ticket;
784
785 /*
786 * this is the pageout path
787 */
788 vnode_size = vnode_pager_get_filesize(vnode_object->vnode_handle);
789
790 if (vnode_size > (offset + PAGE_SIZE)) {
791 /*
792 * preset the maximum size of the cluster
793 * and put us on a nice cluster boundary...
794 * and then clip the size to insure we
795 * don't request past the end of the underlying file
796 */
797 size = PAGE_SIZE * MAX_UPL_TRANSFER;
798 base_offset = offset & ~((signed)(size - 1));
799
800 if ((base_offset + size) > vnode_size)
801 size = round_page_32(((vm_size_t)(vnode_size - base_offset)));
802 } else {
803 /*
804 * we've been requested to page out a page beyond the current
805 * end of the 'file'... don't try to cluster in this case...
806 * we still need to send this page through because it might
807 * be marked precious and the underlying filesystem may need
808 * to do something with it (besides page it out)...
809 */
810 base_offset = offset;
811 size = PAGE_SIZE;
812 }
813 object = memory_object_control_to_vm_object(vnode_object->control_handle);
814
815 if (object == VM_OBJECT_NULL)
816 panic("vnode_pager_cluster_write: NULL vm_object in control handle\n");
817
818 request_flags = UPL_NOBLOCK | UPL_FOR_PAGEOUT | UPL_CLEAN_IN_PLACE |
819 UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM |
820 UPL_SET_INTERNAL | UPL_SET_LITE;
821
822 vm_object_lock(object);
823
824 if ((target_page = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
825 /*
826 * only pick up pages whose ticket number matches
827 * the ticket number of the page orginally targeted
828 * for pageout
829 */
830 ticket = target_page->page_ticket;
831
832 request_flags |= ((ticket << UPL_PAGE_TICKET_SHIFT) & UPL_PAGE_TICKET_MASK);
833 }
834 vm_object_unlock(object);
835
836 vm_object_upl_request(object, base_offset, size,
837 &upl, NULL, NULL, request_flags);
838 if (upl == NULL)
839 panic("vnode_pager_cluster_write: upl request failed\n");
840
841 vnode_pageout(vnode_object->vnode_handle,
842 upl, (vm_offset_t)0, upl->offset, upl->size, UPL_VNODE_PAGER, NULL);
843 }
844 }
845
846
847 /*
848 *
849 */
850 kern_return_t
851 vnode_pager_cluster_read(
852 vnode_pager_t vnode_object,
853 vm_object_offset_t offset,
854 vm_size_t cnt)
855 {
856 int local_error = 0;
857 int kret;
858
859 assert(! (cnt & PAGE_MASK));
860
861 kret = vnode_pagein(vnode_object->vnode_handle,
862 (upl_t) NULL,
863 (vm_offset_t) NULL,
864 offset,
865 cnt,
866 0,
867 &local_error);
868 /*
869 if(kret == PAGER_ABSENT) {
870 Need to work out the defs here, 1 corresponds to PAGER_ABSENT
871 defined in bsd/vm/vm_pager.h However, we should not be including
872 that file here it is a layering violation.
873 */
874 if (kret == 1) {
875 int uplflags;
876 upl_t upl = NULL;
877 unsigned int count = 0;
878 kern_return_t kr;
879
880 uplflags = (UPL_NO_SYNC |
881 UPL_CLEAN_IN_PLACE |
882 UPL_SET_INTERNAL);
883 count = 0;
884 kr = memory_object_upl_request(vnode_object->control_handle,
885 offset, cnt,
886 &upl, NULL, &count, uplflags);
887 if (kr == KERN_SUCCESS) {
888 upl_abort(upl, 0);
889 upl_deallocate(upl);
890 } else {
891 /*
892 * We couldn't gather the page list, probably
893 * because the memory object doesn't have a link
894 * to a VM object anymore (forced unmount, for
895 * example). Just return an error to the vm_fault()
896 * path and let it handle it.
897 */
898 }
899
900 return KERN_FAILURE;
901 }
902
903 return KERN_SUCCESS;
904
905 }
906
907
908 /*
909 *
910 */
911 void
912 vnode_pager_release_from_cache(
913 int *cnt)
914 {
915 memory_object_free_from_cache(
916 &realhost, &vnode_pager_ops, cnt);
917 }
918
919 /*
920 *
921 */
922 vnode_pager_t
923 vnode_object_create(
924 struct vnode *vp)
925 {
926 register vnode_pager_t vnode_object;
927
928 vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
929 if (vnode_object == VNODE_PAGER_NULL)
930 return(VNODE_PAGER_NULL);
931
932 /*
933 * The vm_map call takes both named entry ports and raw memory
934 * objects in the same parameter. We need to make sure that
935 * vm_map does not see this object as a named entry port. So,
936 * we reserve the second word in the object for a fake ip_kotype
937 * setting - that will tell vm_map to use it as a memory object.
938 */
939 vnode_object->pager_ops = &vnode_pager_ops;
940 vnode_object->pager_ikot = IKOT_MEMORY_OBJECT;
941 vnode_object->ref_count = 1;
942 vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL;
943 vnode_object->vnode_handle = vp;
944
945 return(vnode_object);
946 }
947
948 /*
949 *
950 */
951 vnode_pager_t
952 vnode_pager_lookup(
953 memory_object_t name)
954 {
955 vnode_pager_t vnode_object;
956
957 vnode_object = (vnode_pager_t)name;
958 assert(vnode_object->pager_ops == &vnode_pager_ops);
959 return (vnode_object);
960 }
961
962
963 /*********************** proc_info implementation *************/
964
965 #include <sys/bsdtask_info.h>
966
967 static int fill_vnodeinfoforaddr( vm_map_entry_t entry, uint32_t * vnodeaddr, uint32_t * vid);
968
969
970 int
971 fill_procregioninfo(task_t task, uint64_t arg, struct proc_regioninfo_internal *pinfo, uint32_t *vnodeaddr, uint32_t *vid)
972 {
973
974 vm_map_t map = task->map;
975 vm_map_offset_t address = (vm_map_offset_t )arg;
976 vm_map_entry_t tmp_entry;
977 vm_map_entry_t entry;
978 vm_map_offset_t start;
979 vm_region_extended_info_data_t extended;
980 vm_region_top_info_data_t top;
981
982
983 if (map == VM_MAP_NULL)
984 return(0);
985
986 vm_map_lock_read(map);
987
988 start = address;
989 if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
990 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
991 vm_map_unlock_read(map);
992 return(0);
993 }
994 } else {
995 entry = tmp_entry;
996 }
997
998 start = entry->vme_start;
999
1000 pinfo->pri_offset = entry->offset;
1001 pinfo->pri_protection = entry->protection;
1002 pinfo->pri_max_protection = entry->max_protection;
1003 pinfo->pri_inheritance = entry->inheritance;
1004 pinfo->pri_behavior = entry->behavior;
1005 pinfo->pri_user_wired_count = entry->user_wired_count;
1006 pinfo->pri_user_tag = entry->alias;
1007
1008 if (entry->is_sub_map) {
1009 pinfo->pri_flags |= PROC_REGION_SUBMAP;
1010 } else {
1011 if (entry->is_shared)
1012 pinfo->pri_flags |= PROC_REGION_SHARED;
1013 }
1014
1015
1016 extended.protection = entry->protection;
1017 extended.user_tag = entry->alias;
1018 extended.pages_resident = 0;
1019 extended.pages_swapped_out = 0;
1020 extended.pages_shared_now_private = 0;
1021 extended.pages_dirtied = 0;
1022 extended.external_pager = 0;
1023 extended.shadow_depth = 0;
1024
1025 vm_map_region_walk(map, start, entry, entry->offset, entry->vme_end - start, &extended);
1026
1027 if (extended.external_pager && extended.ref_count == 2 && extended.share_mode == SM_SHARED)
1028 extended.share_mode = SM_PRIVATE;
1029
1030 top.private_pages_resident = 0;
1031 top.shared_pages_resident = 0;
1032 vm_map_region_top_walk(entry, &top);
1033
1034
1035 pinfo->pri_pages_resident = extended.pages_resident;
1036 pinfo->pri_pages_shared_now_private = extended.pages_shared_now_private;
1037 pinfo->pri_pages_swapped_out = extended.pages_swapped_out;
1038 pinfo->pri_pages_dirtied = extended.pages_dirtied;
1039 pinfo->pri_ref_count = extended.ref_count;
1040 pinfo->pri_shadow_depth = extended.shadow_depth;
1041 pinfo->pri_share_mode = extended.share_mode;
1042
1043 pinfo->pri_private_pages_resident = top.private_pages_resident;
1044 pinfo->pri_shared_pages_resident = top.shared_pages_resident;
1045 pinfo->pri_obj_id = top.obj_id;
1046
1047 pinfo->pri_address = (uint64_t)start;
1048 pinfo->pri_size = (uint64_t)(entry->vme_end - start);
1049 pinfo->pri_depth = 0;
1050
1051 if ((vnodeaddr != 0) && (entry->is_sub_map == 0)) {
1052 *vnodeaddr = (uint32_t)0;
1053
1054 if (fill_vnodeinfoforaddr(entry, vnodeaddr, vid) ==0) {
1055 vm_map_unlock_read(map);
1056 return(1);
1057 }
1058 }
1059
1060 vm_map_unlock_read(map);
1061 return(1);
1062 }
1063
1064 static int
1065 fill_vnodeinfoforaddr(
1066 vm_map_entry_t entry,
1067 uint32_t * vnodeaddr,
1068 uint32_t * vid)
1069 {
1070 vm_object_t top_object, object;
1071 memory_object_t memory_object;
1072 memory_object_pager_ops_t pager_ops;
1073 kern_return_t kr;
1074 int shadow_depth;
1075
1076
1077 if (entry->is_sub_map) {
1078 return(0);
1079 } else {
1080 /*
1081 * The last object in the shadow chain has the
1082 * relevant pager information.
1083 */
1084 top_object = entry->object.vm_object;
1085 if (top_object == VM_OBJECT_NULL) {
1086 object = VM_OBJECT_NULL;
1087 shadow_depth = 0;
1088 } else {
1089 vm_object_lock(top_object);
1090 for (object = top_object, shadow_depth = 0;
1091 object->shadow != VM_OBJECT_NULL;
1092 object = object->shadow, shadow_depth++) {
1093 vm_object_lock(object->shadow);
1094 vm_object_unlock(object);
1095 }
1096 }
1097 }
1098
1099 if (object == VM_OBJECT_NULL) {
1100 return(0);
1101 } else if (object->internal) {
1102 vm_object_unlock(object);
1103 return(0);
1104 } else if (! object->pager_ready ||
1105 object->terminating ||
1106 ! object->alive) {
1107 vm_object_unlock(object);
1108 return(0);
1109 } else {
1110 memory_object = object->pager;
1111 pager_ops = memory_object->mo_pager_ops;
1112 if (pager_ops == &vnode_pager_ops) {
1113 kr = vnode_pager_get_object_vnode(
1114 memory_object,
1115 vnodeaddr, vid);
1116 if (kr != KERN_SUCCESS) {
1117 vm_object_unlock(object);
1118 return(0);
1119 }
1120 } else {
1121 vm_object_unlock(object);
1122 return(0);
1123 }
1124 }
1125 vm_object_unlock(object);
1126 return(1);
1127 }
1128
1129 kern_return_t
1130 vnode_pager_get_object_vnode (
1131 memory_object_t mem_obj,
1132 uint32_t * vnodeaddr,
1133 uint32_t * vid)
1134 {
1135 vnode_pager_t vnode_object;
1136
1137 vnode_object = vnode_pager_lookup(mem_obj);
1138 if (vnode_object->vnode_handle) {
1139 *vnodeaddr = (uint32_t)vnode_object->vnode_handle;
1140 *vid = (uint32_t)vnode_vid((void *)vnode_object->vnode_handle);
1141
1142 return(KERN_SUCCESS);
1143 }
1144
1145 return(KERN_FAILURE);
1146 }
1147
mirror of XNU