]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/bsd_vm.c
xnu-123.5.tar.gz
[apple/xnu.git] / osfmk / vm / bsd_vm.c
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22
23 #include <sys/errno.h>
24 #include <kern/host.h>
25 #include <mach/mach_types.h>
26 #include <vm/vm_map.h>
27 #include <vm/vm_kern.h>
28 #include <vm/vm_pageout.h>
29 #include <mach/kern_return.h>
30 #include <mach/memory_object_types.h>
31 #include <mach/port.h>
32 #include <mach/policy.h>
33 #include <ipc/ipc_port.h>
34 #include <ipc/ipc_space.h>
35 #include <kern/thread.h>
36 #include <vm/vm_pageout.h>
37
38
39 /* BSD VM COMPONENT INTERFACES */
40 int
41 get_map_nentries(
42 vm_map_t);
43
44 vm_offset_t
45 get_map_start(
46 vm_map_t);
47
48 vm_offset_t
49 get_map_end(
50 vm_map_t);
51
52 /*
53 *
54 */
55 int
56 get_map_nentries(
57 vm_map_t map)
58 {
59 return(map->hdr.nentries);
60 }
61
62 /*
63 *
64 */
65 vm_offset_t
66 get_map_start(
67 vm_map_t map)
68 {
69 return(vm_map_first_entry(map)->vme_start);
70 }
71
72 /*
73 *
74 */
75 vm_offset_t
76 get_map_end(
77 vm_map_t map)
78 {
79 return(vm_map_last_entry(map)->vme_end);
80 }
81
82 /*
83 * BSD VNODE PAGER
84 */
85
86 /* until component support available */
87 int vnode_pager_workaround;
88
89 typedef int vnode_port_t;
90
91 typedef struct vnode_pager {
92 ipc_port_t pager; /* pager */
93 ipc_port_t pager_handle; /* pager handle */
94 ipc_port_t vm_obj_handle; /* memory object's control handle */
95 vnode_port_t vnode_handle; /* vnode handle */
96 } *vnode_pager_t;
97
98 typedef struct vnode_port_entry {
99 queue_chain_t links; /* queue links */
100 ipc_port_t name; /* port name */
101 vnode_pager_t pager_rec; /* pager record */
102 } *vnode_port_entry_t;
103
104
105 #define VNODE_PORT_HASH_COUNT 127
106 #define vnode_port_hash(name_port) \
107 (((int)(name_port) & 0xffffff) % VNODE_PORT_HASH_COUNT)
108
109 queue_head_t vnode_port_hashtable[VNODE_PORT_HASH_COUNT];
110 zone_t vnode_port_hash_zone;
111 decl_simple_lock_data(,vnode_port_hash_lock)
112
113
114 ipc_port_t
115 trigger_name_to_port(
116 mach_port_t);
117
118 void
119 vnode_pager_bootstrap(
120 void);
121
122 void
123 vnode_pager_alloc_map(
124 void);
125
126 ipc_port_t
127 vnode_pager_setup(
128 vnode_port_t,
129 ipc_port_t);
130
131 ipc_port_t
132 vnode_pager_lookup(
133 vnode_port_t,
134 ipc_port_t);
135
136 kern_return_t
137 vnode_pager_init(
138 ipc_port_t,
139 ipc_port_t,
140 vm_size_t);
141
142 kern_return_t
143 vnode_pager_data_request(
144 ipc_port_t,
145 ipc_port_t,
146 vm_object_offset_t,
147 vm_size_t,
148 vm_prot_t);
149
150 kern_return_t
151 vnode_pager_data_return(
152 ipc_port_t,
153 ipc_port_t,
154 vm_object_offset_t,
155 pointer_t,
156 vm_size_t,
157 boolean_t,
158 boolean_t);
159
160 void
161 vnode_pager_no_senders(
162 ipc_port_t,
163 mach_port_mscount_t);
164
165 kern_return_t
166 vnode_pager_terminate(
167 ipc_port_t,
168 ipc_port_t);
169
170 kern_return_t
171 vnode_pager_cluster_read(
172 vnode_pager_t,
173 vm_object_offset_t,
174 vm_size_t);
175
176 void
177 vnode_pager_cluster_write(
178 vnode_pager_t,
179 vm_object_offset_t,
180 vm_size_t);
181
182 kern_return_t
183 memory_object_change_attributes(
184 vm_object_t,
185 memory_object_flavor_t,
186 memory_object_info_t,
187 mach_msg_type_number_t,
188 ipc_port_t,
189 mach_msg_type_name_t);
190
191 int
192 vnode_pagein(
193 vnode_port_t,
194 upl_t,
195 vm_offset_t,
196 vm_object_offset_t,
197 int,
198 int,
199 int *);
200 int
201 vnode_pageout(
202 vnode_port_t,
203 upl_t,
204 vm_offset_t,
205 vm_object_offset_t,
206 int,
207 int,
208 int *);
209
210 vnode_pager_t
211 vnode_object_create(
212 vnode_port_t vp);
213
214 void
215 vnode_port_hash_init(void);
216
217 void
218 vnode_port_hash_insert(
219 ipc_port_t,
220 vnode_pager_t);
221
222 vnode_pager_t
223 vnode_port_hash_lookup(
224 ipc_port_t);
225
226 void
227 vnode_port_hash_delete(
228 ipc_port_t);
229
230 void
231 vnode_pager_release_from_cache(
232 int *cnt);
233
234 zone_t vnode_pager_zone;
235
236
237 #define VNODE_PAGER_NULL ((vnode_pager_t) 0)
238
239 /* TODO: Should be set dynamically by vnode_pager_init() */
240 #define CLUSTER_SHIFT 1
241
242 /* TODO: Should be set dynamically by vnode_pager_bootstrap() */
243 #define MAX_VNODE 10000
244
245
246 #if DEBUG
247 int pagerdebug=0;
248
249 #define PAGER_ALL 0xffffffff
250 #define PAGER_INIT 0x00000001
251 #define PAGER_PAGEIN 0x00000002
252
253 #define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}}
254 #else
255 #define PAGER_DEBUG(LEVEL, A)
256 #endif
257
258 /*
259 * Routine: macx_triggers
260 * Function:
261 * Syscall interface to set the call backs for low and
262 * high water marks.
263 */
264 int
265 macx_triggers(
266 int hi_water,
267 int low_water,
268 int flags,
269 mach_port_t trigger_name)
270 {
271 kern_return_t kr;
272 ipc_port_t default_pager_port = MACH_PORT_NULL;
273 ipc_port_t trigger_port;
274
275 kr = host_default_memory_manager(host_priv_self(),
276 &default_pager_port, 0);
277 if(kr != KERN_SUCCESS) {
278 return EINVAL;
279 }
280 trigger_port = trigger_name_to_port(trigger_name);
281 if(trigger_port == NULL) {
282 return EINVAL;
283 }
284 /* trigger_port is locked and active */
285 ip_unlock(trigger_port);
286 default_pager_triggers(default_pager_port,
287 hi_water, low_water, flags, trigger_port);
288 ipc_port_make_send(trigger_port);
289
290 /*
291 * Set thread scheduling priority and policy for the current thread
292 * it is assumed for the time being that the thread setting the alert
293 * is the same one which will be servicing it.
294 */
295 {
296 struct policy_timeshare_base fifo_base;
297 struct policy_timeshare_limit fifo_limit;
298 policy_base_t base;
299 processor_set_t pset;
300 policy_limit_t limit;
301
302 pset = (current_thread())->processor_set;
303 base = (policy_base_t) &fifo_base;
304 limit = (policy_limit_t) &fifo_limit;
305 fifo_limit.max_priority = fifo_base.base_priority = MAXPRI_STANDARD;
306 thread_set_policy((current_thread())->top_act, pset, POLICY_FIFO, base, POLICY_TIMESHARE_BASE_COUNT, limit, POLICY_TIMESHARE_LIMIT_COUNT);
307 }
308
309 current_thread()->vm_privilege = TRUE;
310 }
311
312 /*
313 *
314 */
315 ipc_port_t
316 trigger_name_to_port(
317 mach_port_t trigger_name)
318 {
319 ipc_port_t trigger_port;
320 ipc_space_t space;
321
322 if (trigger_name == 0)
323 return (NULL);
324
325 space = current_space();
326 if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name,
327 &trigger_port) != KERN_SUCCESS)
328 return (NULL);
329 return trigger_port;
330 }
331
332 /*
333 *
334 */
335 void
336 vnode_pager_bootstrap(void)
337 {
338 register vm_size_t size;
339
340 size = (vm_size_t) sizeof(struct vnode_pager);
341 vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size,
342 PAGE_SIZE, "vnode pager structures");
343 vnode_port_hash_init();
344
345 return;
346 }
347
348 /*
349 *
350 */
351 ipc_port_t
352 vnode_pager_setup(
353 vnode_port_t vp,
354 ipc_port_t pager)
355 {
356 vnode_pager_t vnode_object;
357 kern_return_t kr;
358 ipc_port_t previous;
359
360 if (pager &&
361 (vnode_object = vnode_port_hash_lookup(pager))) {
362 if (vnode_object->vnode_handle == vp)
363 return(pager);
364 }
365
366 vnode_object = vnode_object_create(vp);
367 if (vnode_object == VNODE_PAGER_NULL)
368 panic("vnode_pager_setup: vnode_object_create() failed");
369
370 vnode_object->pager = ipc_port_alloc_kernel();
371 assert (vnode_object->pager != IP_NULL);
372 pager_mux_hash_insert(vnode_object->pager,
373 (rpc_subsystem_t)&vnode_pager_workaround);
374
375 vnode_object->pager_handle = ipc_port_make_send(vnode_object->pager);
376
377 vnode_port_hash_insert(vnode_object->pager_handle, vnode_object);
378
379 ipc_port_make_sonce(vnode_object->pager);
380 ip_lock(vnode_object->pager); /* unlocked in nsrequest below */
381 ipc_port_nsrequest(vnode_object->pager, 1, vnode_object->pager, &previous);
382
383 PAGER_DEBUG(PAGER_INIT, ("vnode_pager_setup: vp %x pager %x vnode_pager %x\n", vp, vnode_object->pager_handle, vnode_object));
384
385 ubc_setpager( vp, vnode_object->pager_handle);
386 return(vnode_object->pager_handle);
387 }
388
389 /*
390 *
391 */
392 ipc_port_t
393 vnode_pager_lookup(
394 vnode_port_t vp,
395 ipc_port_t pager)
396 {
397 vnode_pager_t vnode_object;
398 kern_return_t kr;
399
400 if (pager &&
401 (vnode_object = vnode_port_hash_lookup(pager))) {
402 if (vnode_object->vnode_handle == vp)
403 return(vnode_object->vm_obj_handle);
404 else
405 return NULL;
406 }
407 else
408 return NULL;
409 }
410
411 /*
412 *
413 */
414 kern_return_t
415 vnode_pager_init(ipc_port_t pager,
416 ipc_port_t pager_request,
417 vm_size_t pg_size)
418 {
419 vnode_pager_t vnode_object;
420 kern_return_t kr;
421 memory_object_attr_info_data_t attributes;
422 vm_object_t vm_object;
423
424
425 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %x, %x, %x\n", pager, pager_request, pg_size));
426
427 vnode_object = vnode_port_hash_lookup(pager);
428 if (vnode_object == VNODE_PAGER_NULL)
429 panic("vnode_pager_init: lookup failed");
430
431 vnode_object->vm_obj_handle = pager_request;
432
433 vm_object = vm_object_lookup(pager_request);
434
435 if (vm_object == VM_OBJECT_NULL)
436 panic("vnode_pager_init: vm_object_lookup() failed");
437
438 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
439 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
440 attributes.cluster_size = (1 << (PAGE_SHIFT));
441 attributes.may_cache_object = TRUE;
442 attributes.temporary = TRUE;
443
444 kr = memory_object_change_attributes(
445 vm_object,
446 MEMORY_OBJECT_ATTRIBUTE_INFO,
447 (memory_object_info_t) &attributes,
448 MEMORY_OBJECT_ATTR_INFO_COUNT,
449 MACH_PORT_NULL, 0);
450 if (kr != KERN_SUCCESS)
451 panic("vnode_pager_init: memory_object_change_attributes() failed");
452
453 return(KERN_SUCCESS);
454 }
455
456 /*
457 *
458 */
459 kern_return_t
460 vnode_pager_data_return(
461 ipc_port_t mem_obj,
462 ipc_port_t control_port,
463 vm_object_offset_t offset,
464 pointer_t addr,
465 vm_size_t data_cnt,
466 boolean_t dirty,
467 boolean_t kernel_copy)
468 {
469 register vnode_pager_t vnode_object;
470
471 vnode_object = vnode_port_hash_lookup(mem_obj);
472 if (vnode_object == VNODE_PAGER_NULL)
473 panic("vnode_pager_data_return: lookup failed");
474
475 vnode_pager_cluster_write(vnode_object, offset, data_cnt);
476
477 return KERN_SUCCESS;
478 }
479
480 /*
481 *
482 */
483 kern_return_t
484 vnode_pager_data_request(
485 ipc_port_t mem_obj,
486 ipc_port_t mem_obj_control,
487 vm_object_offset_t offset,
488 vm_size_t length,
489 vm_prot_t protection_required)
490 {
491 register vnode_pager_t vnode_object;
492
493 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x, %x\n", mem_obj, mem_obj_control, offset, length, protection_required));
494
495 vnode_object = vnode_port_hash_lookup(mem_obj);
496
497 PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, %x, vnode_object %x\n", mem_obj, mem_obj_control, offset, length, protection_required, vnode_object));
498
499 if (vnode_object == VNODE_PAGER_NULL)
500 panic("vnode_pager_data_request: lookup failed");
501
502 vnode_pager_cluster_read(vnode_object, offset, length);
503
504 return KERN_SUCCESS;
505 }
506
507 /*
508 *
509 */
510 void
511 vnode_pager_no_senders(
512 ipc_port_t mem_obj,
513 mach_port_mscount_t mscount)
514 {
515 register vnode_pager_t vnode_object;
516
517 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_nosenders: %x, %x\n", mem_obj, mscount));
518
519 vnode_object = vnode_port_hash_lookup(mem_obj);
520 if (vnode_object == VNODE_PAGER_NULL)
521 panic("vnode_pager_no_senders: lookup failed");
522
523 assert(vnode_object->pager_handle == mem_obj);
524
525 pager_mux_hash_delete((ipc_port_t) vnode_object->pager_handle);
526 ipc_port_dealloc_kernel(vnode_object->pager);
527 vnode_port_hash_delete(vnode_object->pager_handle);
528 if (vnode_object->vnode_handle != (vnode_port_t) NULL) {
529 vnode_pager_vrele(vnode_object->vnode_handle);
530 }
531 zfree(vnode_pager_zone, (vm_offset_t) vnode_object);
532
533 return;
534 }
535
536 /*
537 *
538 */
539 kern_return_t
540 vnode_pager_terminate(
541 ipc_port_t mem_obj,
542 ipc_port_t mem_obj_control)
543 {
544 register vnode_pager_t vnode_object;
545
546 PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x, %x\n", mem_obj, mem_obj_control));
547
548 vnode_object = vnode_port_hash_lookup(mem_obj);
549 if (vnode_object == VNODE_PAGER_NULL)
550 panic("vnode_pager_terminate: lookup failed");
551
552 assert(vnode_object->pager_handle == mem_obj);
553
554 /* release extra send right created by the fact that the caller */
555 /* of vnode_pager_setup does not establish a mapping between a */
556 /* cache object and the mem_obj (AMO). When a subsequent vm_map */
557 /* is done, vm_map will bump the send right count */
558 ipc_port_release_send(mem_obj);
559
560 /* release a send right because terminate is called directly and */
561 /* not through IPC, the right won't disappear quietly */
562 ipc_port_release_send(mem_obj);
563
564 ipc_port_dealloc_kernel(mem_obj_control);
565
566 return(KERN_SUCCESS);
567 }
568
569 /*
570 *
571 */
572 kern_return_t
573 vnode_pager_synchronize(
574 ipc_port_t pager,
575 ipc_port_t pager_request,
576 vm_object_offset_t offset,
577 vm_offset_t length,
578 vm_sync_t sync_flags)
579 {
580 memory_object_synchronize_completed(vm_object_lookup(pager_request), offset, length);
581
582 return (KERN_SUCCESS);
583 }
584
585 /*
586 *
587 */
588 void
589 vnode_pager_cluster_write(
590 vnode_pager_t vnode_object,
591 vm_object_offset_t offset,
592 vm_size_t cnt)
593 {
594 int error = 0;
595 int local_error = 0;
596 int kret;
597 int size;
598
599 if (cnt & PAGE_MASK) {
600 panic("vs_cluster_write: cnt not a multiple of PAGE_SIZE");
601 }
602 size = (cnt < (PAGE_SIZE*32)) ? cnt : (PAGE_SIZE*32); /* effective min */
603
604 while (cnt) {
605
606 kret = vnode_pageout(vnode_object->vnode_handle, (upl_t )NULL, (vm_offset_t)NULL, offset, size, 0, &local_error);
607
608 if (local_error != 0) {
609 error = local_error;
610 local_error = 0;
611 }
612 cnt -= size;
613 offset += size;
614 }
615 #if 0
616 if (error != 0)
617 return(KERN_FAILURE);
618
619 return(KERN_SUCCESS);
620 #endif /* 0 */
621 }
622
623
624 /*
625 *
626 */
627 kern_return_t
628 vnode_pager_cluster_read(
629 vnode_pager_t vnode_object,
630 vm_object_offset_t offset,
631 vm_size_t cnt)
632 {
633 int error = 0;
634 int local_error = 0;
635 int kret;
636 int size;
637
638 if(cnt & PAGE_MASK) {
639 panic("vs_cluster_read: cnt not a multiple of PAGE_SIZE");
640 }
641
642 size = PAGE_SIZE;
643
644 while (cnt) {
645
646 kret = vnode_pagein(vnode_object->vnode_handle, (upl_t)NULL, (vm_offset_t)NULL, offset, size, 0, &local_error);
647
648 if (local_error != 0) {
649 error = local_error;
650 local_error = 0;
651 }
652 cnt -= size;
653 offset += size;
654 }
655 if (error != 0)
656 return(KERN_FAILURE);
657
658 return(KERN_SUCCESS);
659
660 }
661
662
663 /*
664 *
665 */
666 void
667 vnode_pager_release_from_cache(
668 int *cnt)
669 {
670 memory_object_free_from_cache(
671 &realhost, (int)&vnode_pager_workaround, cnt);
672 }
673
674 /*
675 *
676 */
677 vnode_pager_t
678 vnode_object_create(
679 vnode_port_t vp)
680 {
681 register vnode_pager_t vnode_object;
682
683 vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone);
684 if (vnode_object == VNODE_PAGER_NULL)
685 return(VNODE_PAGER_NULL);
686 vnode_object->pager_handle = IP_NULL;
687 vnode_object->vm_obj_handle = IP_NULL;
688 vnode_object->vnode_handle = vp;
689
690 return(vnode_object);
691 }
692
693 /*
694 *
695 */
696 void
697 vnode_port_hash_init(void)
698 {
699 register vm_size_t size;
700 register int i;
701
702
703 size = (vm_size_t) sizeof(struct vnode_port_entry);
704
705 vnode_port_hash_zone = zinit(size,
706 (vm_size_t) MAX_VNODE * size,
707 PAGE_SIZE, "vnode_pager port hash");
708
709 for (i = 0; i < VNODE_PORT_HASH_COUNT; i++)
710 queue_init(&vnode_port_hashtable[i]);
711
712 simple_lock_init(&vnode_port_hash_lock,ETAP_NO_TRACE);
713 }
714
715 /*
716 *
717 */
718 void
719 vnode_port_hash_insert(
720 ipc_port_t name_port,
721 vnode_pager_t rec)
722 {
723 register vnode_port_entry_t new_entry;
724
725 new_entry = (vnode_port_entry_t) zalloc(vnode_port_hash_zone);
726 /*
727 * TODO: Delete the following check once MAX_VNODE is removed
728 */
729 if (!new_entry)
730 panic("vnode_port_hash_insert: no space");
731 new_entry->name = name_port;
732 new_entry->pager_rec = rec;
733
734 simple_lock(&vnode_port_hash_lock);
735 queue_enter(&vnode_port_hashtable[vnode_port_hash(name_port)],
736 new_entry, vnode_port_entry_t, links);
737 simple_unlock(&vnode_port_hash_lock);
738 }
739
740 /*
741 *
742 */
743 vnode_pager_t
744 vnode_port_hash_lookup(
745 ipc_port_t name_port)
746 {
747 register queue_t bucket;
748 register vnode_port_entry_t entry;
749 vnode_pager_t rec;
750
751 bucket = (queue_t) &vnode_port_hashtable[vnode_port_hash(name_port)];
752
753 simple_lock(&vnode_port_hash_lock);
754 entry = (vnode_port_entry_t) queue_first(bucket);
755 while (!queue_end(bucket,&entry->links)) {
756 if (entry->name == name_port) {
757 rec = entry->pager_rec;
758 simple_unlock(&vnode_port_hash_lock);
759 return(rec);
760 }
761 entry = (vnode_port_entry_t)queue_next(&entry->links);
762 }
763 simple_unlock(&vnode_port_hash_lock);
764 return(VNODE_PAGER_NULL);
765 }
766
767 /*
768 *
769 */
770 void
771 vnode_port_hash_delete(
772 ipc_port_t name_port)
773 {
774 register queue_t bucket;
775 register vnode_port_entry_t entry;
776
777 bucket = (queue_t) &vnode_port_hashtable[vnode_port_hash(name_port)];
778
779 simple_lock(&vnode_port_hash_lock);
780 entry = (vnode_port_entry_t) queue_first(bucket);
781 while (!queue_end(bucket,&entry->links)) {
782 if (entry->name == name_port) {
783 queue_remove(bucket, entry, vnode_port_entry_t,links);
784 simple_unlock(&vnode_port_hash_lock);
785 zfree(vnode_port_hash_zone, (vm_offset_t) entry);
786 return;
787 }
788 entry = (vnode_port_entry_t)queue_next(&entry->links);
789 }
790 simple_unlock(&vnode_port_hash_lock);
791 }