]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved. | |
3 | * | |
4 | * @APPLE_LICENSE_HEADER_START@ | |
5 | * | |
6 | * The contents of this file constitute Original Code as defined in and | |
7 | * are subject to the Apple Public Source License Version 1.1 (the | |
8 | * "License"). You may not use this file except in compliance with the | |
9 | * License. Please obtain a copy of the License at | |
10 | * http://www.apple.com/publicsource and read it before using this file. | |
11 | * | |
12 | * This Original Code and all software distributed under the License are | |
13 | * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
14 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, | |
15 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the | |
17 | * License for the specific language governing rights and limitations | |
18 | * under the License. | |
19 | * | |
20 | * @APPLE_LICENSE_HEADER_END@ | |
21 | */ | |
22 | ||
23 | #include <sys/errno.h> | |
24 | ||
25 | #include <mach/mach_types.h> | |
26 | #include <mach/mach_traps.h> | |
27 | #include <mach/host_priv.h> | |
28 | #include <mach/kern_return.h> | |
29 | #include <mach/memory_object_control.h> | |
30 | #include <mach/memory_object_types.h> | |
31 | #include <mach/port.h> | |
32 | #include <mach/policy.h> | |
33 | #include <mach/upl.h> | |
34 | #include <mach/thread_act.h> | |
35 | ||
36 | #include <kern/host.h> | |
37 | #include <kern/thread.h> | |
38 | ||
39 | #include <ipc/ipc_port.h> | |
40 | #include <ipc/ipc_space.h> | |
41 | ||
42 | #include <default_pager/default_pager_types.h> | |
43 | #include <default_pager/default_pager_object_server.h> | |
44 | ||
45 | #include <vm/vm_map.h> | |
46 | #include <vm/vm_kern.h> | |
47 | #include <vm/vm_pageout.h> | |
48 | #include <vm/memory_object.h> | |
49 | #include <vm/vm_pageout.h> | |
50 | #include <vm/vm_protos.h> | |
51 | ||
52 | /* BSD VM COMPONENT INTERFACES */ | |
53 | int | |
54 | get_map_nentries( | |
55 | vm_map_t); | |
56 | ||
57 | vm_offset_t | |
58 | get_map_start( | |
59 | vm_map_t); | |
60 | ||
61 | vm_offset_t | |
62 | get_map_end( | |
63 | vm_map_t); | |
64 | ||
65 | /* | |
66 | * | |
67 | */ | |
68 | int | |
69 | get_map_nentries( | |
70 | vm_map_t map) | |
71 | { | |
72 | return(map->hdr.nentries); | |
73 | } | |
74 | ||
75 | mach_vm_offset_t | |
76 | mach_get_vm_start(vm_map_t map) | |
77 | { | |
78 | return( vm_map_first_entry(map)->vme_start); | |
79 | } | |
80 | ||
81 | mach_vm_offset_t | |
82 | mach_get_vm_end(vm_map_t map) | |
83 | { | |
84 | return( vm_map_last_entry(map)->vme_end); | |
85 | } | |
86 | ||
87 | /* | |
88 | * Legacy routines to get the start and end for a vm_map_t. They | |
89 | * return them in the vm_offset_t format. So, they should only be | |
90 | * called on maps that are the same size as the kernel map for | |
91 | * accurate results. | |
92 | */ | |
93 | vm_offset_t | |
94 | get_vm_start( | |
95 | vm_map_t map) | |
96 | { | |
97 | return(CAST_DOWN(vm_offset_t, vm_map_first_entry(map)->vme_start)); | |
98 | } | |
99 | ||
100 | vm_offset_t | |
101 | get_vm_end( | |
102 | vm_map_t map) | |
103 | { | |
104 | return(CAST_DOWN(vm_offset_t, vm_map_last_entry(map)->vme_end)); | |
105 | } | |
106 | ||
107 | /* | |
108 | * BSD VNODE PAGER | |
109 | */ | |
110 | ||
111 | /* until component support available */ | |
112 | int vnode_pager_workaround; | |
113 | ||
114 | typedef struct vnode_pager { | |
115 | int *pager; /* pager workaround pointer */ | |
116 | unsigned int pager_ikot; /* JMM: fake ip_kotype() */ | |
117 | unsigned int ref_count; /* reference count */ | |
118 | memory_object_control_t control_handle; /* mem object control handle */ | |
119 | struct vnode *vnode_handle; /* vnode handle */ | |
120 | } *vnode_pager_t; | |
121 | ||
122 | ||
123 | ipc_port_t | |
124 | trigger_name_to_port( /* forward */ | |
125 | mach_port_t); | |
126 | ||
127 | kern_return_t | |
128 | vnode_pager_cluster_read( /* forward */ | |
129 | vnode_pager_t, | |
130 | vm_object_offset_t, | |
131 | vm_size_t); | |
132 | ||
133 | void | |
134 | vnode_pager_cluster_write( /* forward */ | |
135 | vnode_pager_t, | |
136 | vm_object_offset_t, | |
137 | vm_size_t, | |
138 | vm_object_offset_t *, | |
139 | int *, | |
140 | int); | |
141 | ||
142 | ||
143 | vnode_pager_t | |
144 | vnode_object_create( /* forward */ | |
145 | struct vnode *); | |
146 | ||
147 | vnode_pager_t | |
148 | vnode_pager_lookup( /* forward */ | |
149 | memory_object_t); | |
150 | ||
151 | zone_t vnode_pager_zone; | |
152 | ||
153 | ||
154 | #define VNODE_PAGER_NULL ((vnode_pager_t) 0) | |
155 | ||
156 | /* TODO: Should be set dynamically by vnode_pager_init() */ | |
157 | #define CLUSTER_SHIFT 1 | |
158 | ||
159 | /* TODO: Should be set dynamically by vnode_pager_bootstrap() */ | |
160 | #define MAX_VNODE 10000 | |
161 | ||
162 | ||
163 | #if DEBUG | |
164 | int pagerdebug=0; | |
165 | ||
166 | #define PAGER_ALL 0xffffffff | |
167 | #define PAGER_INIT 0x00000001 | |
168 | #define PAGER_PAGEIN 0x00000002 | |
169 | ||
170 | #define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}} | |
171 | #else | |
172 | #define PAGER_DEBUG(LEVEL, A) | |
173 | #endif | |
174 | ||
175 | /* | |
176 | * Routine: macx_triggers | |
177 | * Function: | |
178 | * Syscall interface to set the call backs for low and | |
179 | * high water marks. | |
180 | */ | |
181 | int | |
182 | macx_triggers( | |
183 | struct macx_triggers_args *args) | |
184 | { | |
185 | int hi_water = args->hi_water; | |
186 | int low_water = args->low_water; | |
187 | int flags = args->flags; | |
188 | mach_port_t trigger_name = args->alert_port; | |
189 | kern_return_t kr; | |
190 | memory_object_default_t default_pager; | |
191 | ipc_port_t trigger_port; | |
192 | ||
193 | default_pager = MEMORY_OBJECT_DEFAULT_NULL; | |
194 | kr = host_default_memory_manager(host_priv_self(), | |
195 | &default_pager, 0); | |
196 | if(kr != KERN_SUCCESS) { | |
197 | return EINVAL; | |
198 | } | |
199 | ||
200 | if ((flags & SWAP_ENCRYPT_ON) && | |
201 | (flags & SWAP_ENCRYPT_OFF)) { | |
202 | /* can't have it both ways */ | |
203 | return EINVAL; | |
204 | } | |
205 | ||
206 | if (flags & SWAP_ENCRYPT_ON) { | |
207 | /* ENCRYPTED SWAP: tell default_pager to encrypt */ | |
208 | default_pager_triggers(default_pager, | |
209 | 0, 0, | |
210 | SWAP_ENCRYPT_ON, | |
211 | IP_NULL); | |
212 | } else if (flags & SWAP_ENCRYPT_OFF) { | |
213 | /* ENCRYPTED SWAP: tell default_pager not to encrypt */ | |
214 | default_pager_triggers(default_pager, | |
215 | 0, 0, | |
216 | SWAP_ENCRYPT_OFF, | |
217 | IP_NULL); | |
218 | } | |
219 | ||
220 | if (flags & HI_WAT_ALERT) { | |
221 | trigger_port = trigger_name_to_port(trigger_name); | |
222 | if(trigger_port == NULL) { | |
223 | return EINVAL; | |
224 | } | |
225 | /* trigger_port is locked and active */ | |
226 | ipc_port_make_send_locked(trigger_port); | |
227 | /* now unlocked */ | |
228 | default_pager_triggers(default_pager, | |
229 | hi_water, low_water, | |
230 | HI_WAT_ALERT, trigger_port); | |
231 | } | |
232 | ||
233 | if (flags & LO_WAT_ALERT) { | |
234 | trigger_port = trigger_name_to_port(trigger_name); | |
235 | if(trigger_port == NULL) { | |
236 | return EINVAL; | |
237 | } | |
238 | /* trigger_port is locked and active */ | |
239 | ipc_port_make_send_locked(trigger_port); | |
240 | /* and now its unlocked */ | |
241 | default_pager_triggers(default_pager, | |
242 | hi_water, low_water, | |
243 | LO_WAT_ALERT, trigger_port); | |
244 | } | |
245 | ||
246 | /* | |
247 | * Set thread scheduling priority and policy for the current thread | |
248 | * it is assumed for the time being that the thread setting the alert | |
249 | * is the same one which will be servicing it. | |
250 | * | |
251 | * XXX This does not belong in the kernel XXX | |
252 | */ | |
253 | { | |
254 | thread_precedence_policy_data_t pre; | |
255 | thread_extended_policy_data_t ext; | |
256 | ||
257 | ext.timeshare = FALSE; | |
258 | pre.importance = INT32_MAX; | |
259 | ||
260 | thread_policy_set(current_thread(), | |
261 | THREAD_EXTENDED_POLICY, | |
262 | (thread_policy_t)&ext, | |
263 | THREAD_EXTENDED_POLICY_COUNT); | |
264 | ||
265 | thread_policy_set(current_thread(), | |
266 | THREAD_PRECEDENCE_POLICY, | |
267 | (thread_policy_t)&pre, | |
268 | THREAD_PRECEDENCE_POLICY_COUNT); | |
269 | } | |
270 | ||
271 | current_thread()->options |= TH_OPT_VMPRIV; | |
272 | ||
273 | return 0; | |
274 | } | |
275 | ||
276 | /* | |
277 | * | |
278 | */ | |
279 | ipc_port_t | |
280 | trigger_name_to_port( | |
281 | mach_port_t trigger_name) | |
282 | { | |
283 | ipc_port_t trigger_port; | |
284 | ipc_space_t space; | |
285 | ||
286 | if (trigger_name == 0) | |
287 | return (NULL); | |
288 | ||
289 | space = current_space(); | |
290 | if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name, | |
291 | &trigger_port) != KERN_SUCCESS) | |
292 | return (NULL); | |
293 | return trigger_port; | |
294 | } | |
295 | ||
296 | ||
297 | extern int uiomove64(addr64_t, int, void *); | |
298 | #define MAX_RUN 32 | |
299 | ||
300 | int | |
301 | memory_object_control_uiomove( | |
302 | memory_object_control_t control, | |
303 | memory_object_offset_t offset, | |
304 | void * uio, | |
305 | int start_offset, | |
306 | int io_requested, | |
307 | int mark_dirty) | |
308 | { | |
309 | vm_object_t object; | |
310 | vm_page_t dst_page; | |
311 | int xsize; | |
312 | int retval = 0; | |
313 | int cur_run; | |
314 | int cur_needed; | |
315 | int i; | |
316 | vm_page_t page_run[MAX_RUN]; | |
317 | ||
318 | ||
319 | object = memory_object_control_to_vm_object(control); | |
320 | if (object == VM_OBJECT_NULL) { | |
321 | return (0); | |
322 | } | |
323 | assert(!object->internal); | |
324 | ||
325 | vm_object_lock(object); | |
326 | ||
327 | if (mark_dirty && object->copy != VM_OBJECT_NULL) { | |
328 | /* | |
329 | * We can't modify the pages without honoring | |
330 | * copy-on-write obligations first, so fall off | |
331 | * this optimized path and fall back to the regular | |
332 | * path. | |
333 | */ | |
334 | vm_object_unlock(object); | |
335 | return 0; | |
336 | } | |
337 | ||
338 | while (io_requested && retval == 0) { | |
339 | ||
340 | cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE; | |
341 | ||
342 | if (cur_needed > MAX_RUN) | |
343 | cur_needed = MAX_RUN; | |
344 | ||
345 | for (cur_run = 0; cur_run < cur_needed; ) { | |
346 | ||
347 | if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL) | |
348 | break; | |
349 | /* | |
350 | * Sync up on getting the busy bit | |
351 | */ | |
352 | if ((dst_page->busy || dst_page->cleaning)) { | |
353 | /* | |
354 | * someone else is playing with the page... if we've | |
355 | * already collected pages into this run, go ahead | |
356 | * and process now, we can't block on this | |
357 | * page while holding other pages in the BUSY state | |
358 | * otherwise we will wait | |
359 | */ | |
360 | if (cur_run) | |
361 | break; | |
362 | PAGE_SLEEP(object, dst_page, THREAD_UNINT); | |
363 | continue; | |
364 | } | |
365 | /* | |
366 | * this routine is only called when copying | |
367 | * to/from real files... no need to consider | |
368 | * encrypted swap pages | |
369 | */ | |
370 | assert(!dst_page->encrypted); | |
371 | ||
372 | if (mark_dirty) | |
373 | dst_page->dirty = TRUE; | |
374 | dst_page->busy = TRUE; | |
375 | ||
376 | page_run[cur_run++] = dst_page; | |
377 | ||
378 | offset += PAGE_SIZE_64; | |
379 | } | |
380 | if (cur_run == 0) | |
381 | /* | |
382 | * we hit a 'hole' in the cache | |
383 | * we bail at this point | |
384 | * we'll unlock the object below | |
385 | */ | |
386 | break; | |
387 | vm_object_unlock(object); | |
388 | ||
389 | for (i = 0; i < cur_run; i++) { | |
390 | ||
391 | dst_page = page_run[i]; | |
392 | ||
393 | if ((xsize = PAGE_SIZE - start_offset) > io_requested) | |
394 | xsize = io_requested; | |
395 | ||
396 | if ( (retval = uiomove64((addr64_t)(((addr64_t)(dst_page->phys_page) << 12) + start_offset), xsize, uio)) ) | |
397 | break; | |
398 | ||
399 | io_requested -= xsize; | |
400 | start_offset = 0; | |
401 | } | |
402 | vm_object_lock(object); | |
403 | ||
404 | for (i = 0; i < cur_run; i++) { | |
405 | dst_page = page_run[i]; | |
406 | ||
407 | PAGE_WAKEUP_DONE(dst_page); | |
408 | } | |
409 | } | |
410 | vm_object_unlock(object); | |
411 | ||
412 | return (retval); | |
413 | } | |
414 | ||
415 | ||
416 | /* | |
417 | * | |
418 | */ | |
419 | void | |
420 | vnode_pager_bootstrap(void) | |
421 | { | |
422 | register vm_size_t size; | |
423 | ||
424 | size = (vm_size_t) sizeof(struct vnode_pager); | |
425 | vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size, | |
426 | PAGE_SIZE, "vnode pager structures"); | |
427 | return; | |
428 | } | |
429 | ||
430 | /* | |
431 | * | |
432 | */ | |
433 | memory_object_t | |
434 | vnode_pager_setup( | |
435 | struct vnode *vp, | |
436 | __unused memory_object_t pager) | |
437 | { | |
438 | vnode_pager_t vnode_object; | |
439 | ||
440 | vnode_object = vnode_object_create(vp); | |
441 | if (vnode_object == VNODE_PAGER_NULL) | |
442 | panic("vnode_pager_setup: vnode_object_create() failed"); | |
443 | return((memory_object_t)vnode_object); | |
444 | } | |
445 | ||
446 | /* | |
447 | * | |
448 | */ | |
449 | kern_return_t | |
450 | vnode_pager_init(memory_object_t mem_obj, | |
451 | memory_object_control_t control, | |
452 | #if !DEBUG | |
453 | __unused | |
454 | #endif | |
455 | vm_size_t pg_size) | |
456 | { | |
457 | vnode_pager_t vnode_object; | |
458 | kern_return_t kr; | |
459 | memory_object_attr_info_data_t attributes; | |
460 | ||
461 | ||
462 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %p, %p, %x\n", mem_obj, control, pg_size)); | |
463 | ||
464 | if (control == MEMORY_OBJECT_CONTROL_NULL) | |
465 | return KERN_INVALID_ARGUMENT; | |
466 | ||
467 | vnode_object = vnode_pager_lookup(mem_obj); | |
468 | ||
469 | memory_object_control_reference(control); | |
470 | ||
471 | vnode_object->control_handle = control; | |
472 | ||
473 | attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; | |
474 | /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ | |
475 | attributes.cluster_size = (1 << (PAGE_SHIFT)); | |
476 | attributes.may_cache_object = TRUE; | |
477 | attributes.temporary = TRUE; | |
478 | ||
479 | kr = memory_object_change_attributes( | |
480 | control, | |
481 | MEMORY_OBJECT_ATTRIBUTE_INFO, | |
482 | (memory_object_info_t) &attributes, | |
483 | MEMORY_OBJECT_ATTR_INFO_COUNT); | |
484 | if (kr != KERN_SUCCESS) | |
485 | panic("vnode_pager_init: memory_object_change_attributes() failed"); | |
486 | ||
487 | return(KERN_SUCCESS); | |
488 | } | |
489 | ||
490 | /* | |
491 | * | |
492 | */ | |
493 | kern_return_t | |
494 | vnode_pager_data_return( | |
495 | memory_object_t mem_obj, | |
496 | memory_object_offset_t offset, | |
497 | vm_size_t data_cnt, | |
498 | memory_object_offset_t *resid_offset, | |
499 | int *io_error, | |
500 | __unused boolean_t dirty, | |
501 | __unused boolean_t kernel_copy, | |
502 | int upl_flags) | |
503 | { | |
504 | register vnode_pager_t vnode_object; | |
505 | ||
506 | vnode_object = vnode_pager_lookup(mem_obj); | |
507 | ||
508 | vnode_pager_cluster_write(vnode_object, offset, data_cnt, resid_offset, io_error, upl_flags); | |
509 | ||
510 | return KERN_SUCCESS; | |
511 | } | |
512 | ||
513 | kern_return_t | |
514 | vnode_pager_data_initialize( | |
515 | __unused memory_object_t mem_obj, | |
516 | __unused memory_object_offset_t offset, | |
517 | __unused vm_size_t data_cnt) | |
518 | { | |
519 | panic("vnode_pager_data_initialize"); | |
520 | return KERN_FAILURE; | |
521 | } | |
522 | ||
523 | kern_return_t | |
524 | vnode_pager_data_unlock( | |
525 | __unused memory_object_t mem_obj, | |
526 | __unused memory_object_offset_t offset, | |
527 | __unused vm_size_t size, | |
528 | __unused vm_prot_t desired_access) | |
529 | { | |
530 | return KERN_FAILURE; | |
531 | } | |
532 | ||
533 | kern_return_t | |
534 | vnode_pager_get_object_size( | |
535 | memory_object_t mem_obj, | |
536 | memory_object_offset_t *length) | |
537 | { | |
538 | vnode_pager_t vnode_object; | |
539 | ||
540 | vnode_object = vnode_pager_lookup(mem_obj); | |
541 | ||
542 | *length = vnode_pager_get_filesize(vnode_object->vnode_handle); | |
543 | return KERN_SUCCESS; | |
544 | } | |
545 | ||
546 | /* | |
547 | * | |
548 | */ | |
549 | kern_return_t | |
550 | vnode_pager_data_request( | |
551 | memory_object_t mem_obj, | |
552 | memory_object_offset_t offset, | |
553 | vm_size_t length, | |
554 | #if !DEBUG | |
555 | __unused | |
556 | #endif | |
557 | vm_prot_t protection_required) | |
558 | { | |
559 | register vnode_pager_t vnode_object; | |
560 | ||
561 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required)); | |
562 | ||
563 | vnode_object = vnode_pager_lookup(mem_obj); | |
564 | ||
565 | PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object)); | |
566 | ||
567 | return vnode_pager_cluster_read(vnode_object, offset, length); | |
568 | } | |
569 | ||
570 | /* | |
571 | * | |
572 | */ | |
573 | void | |
574 | vnode_pager_reference( | |
575 | memory_object_t mem_obj) | |
576 | { | |
577 | register vnode_pager_t vnode_object; | |
578 | unsigned int new_ref_count; | |
579 | ||
580 | vnode_object = vnode_pager_lookup(mem_obj); | |
581 | new_ref_count = hw_atomic_add(&vnode_object->ref_count, 1); | |
582 | assert(new_ref_count > 1); | |
583 | } | |
584 | ||
585 | /* | |
586 | * | |
587 | */ | |
588 | void | |
589 | vnode_pager_deallocate( | |
590 | memory_object_t mem_obj) | |
591 | { | |
592 | register vnode_pager_t vnode_object; | |
593 | ||
594 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj)); | |
595 | ||
596 | vnode_object = vnode_pager_lookup(mem_obj); | |
597 | ||
598 | if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) { | |
599 | if (vnode_object->vnode_handle != NULL) { | |
600 | vnode_pager_vrele(vnode_object->vnode_handle); | |
601 | } | |
602 | zfree(vnode_pager_zone, vnode_object); | |
603 | } | |
604 | return; | |
605 | } | |
606 | ||
607 | /* | |
608 | * | |
609 | */ | |
610 | kern_return_t | |
611 | vnode_pager_terminate( | |
612 | #if !DEBUG | |
613 | __unused | |
614 | #endif | |
615 | memory_object_t mem_obj) | |
616 | { | |
617 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj)); | |
618 | ||
619 | return(KERN_SUCCESS); | |
620 | } | |
621 | ||
622 | /* | |
623 | * | |
624 | */ | |
625 | kern_return_t | |
626 | vnode_pager_synchronize( | |
627 | memory_object_t mem_obj, | |
628 | memory_object_offset_t offset, | |
629 | vm_size_t length, | |
630 | __unused vm_sync_t sync_flags) | |
631 | { | |
632 | register vnode_pager_t vnode_object; | |
633 | ||
634 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj)); | |
635 | ||
636 | vnode_object = vnode_pager_lookup(mem_obj); | |
637 | ||
638 | memory_object_synchronize_completed(vnode_object->control_handle, offset, length); | |
639 | ||
640 | return (KERN_SUCCESS); | |
641 | } | |
642 | ||
643 | /* | |
644 | * | |
645 | */ | |
646 | kern_return_t | |
647 | vnode_pager_unmap( | |
648 | memory_object_t mem_obj) | |
649 | { | |
650 | register vnode_pager_t vnode_object; | |
651 | ||
652 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj)); | |
653 | ||
654 | vnode_object = vnode_pager_lookup(mem_obj); | |
655 | ||
656 | ubc_unmap(vnode_object->vnode_handle); | |
657 | return KERN_SUCCESS; | |
658 | } | |
659 | ||
660 | ||
661 | /* | |
662 | * | |
663 | */ | |
664 | void | |
665 | vnode_pager_cluster_write( | |
666 | vnode_pager_t vnode_object, | |
667 | vm_object_offset_t offset, | |
668 | vm_size_t cnt, | |
669 | vm_object_offset_t * resid_offset, | |
670 | int * io_error, | |
671 | int upl_flags) | |
672 | { | |
673 | vm_size_t size; | |
674 | upl_t upl = NULL; | |
675 | int request_flags; | |
676 | int errno; | |
677 | ||
678 | if (upl_flags & UPL_MSYNC) { | |
679 | ||
680 | upl_flags |= UPL_VNODE_PAGER; | |
681 | ||
682 | if ( (upl_flags & UPL_IOSYNC) && io_error) | |
683 | upl_flags |= UPL_KEEPCACHED; | |
684 | ||
685 | while (cnt) { | |
686 | kern_return_t kr; | |
687 | ||
688 | size = (cnt < (PAGE_SIZE * MAX_UPL_TRANSFER)) ? cnt : (PAGE_SIZE * MAX_UPL_TRANSFER); /* effective max */ | |
689 | ||
690 | request_flags = UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | UPL_CLEAN_IN_PLACE | | |
691 | UPL_SET_INTERNAL | UPL_SET_LITE; | |
692 | ||
693 | kr = memory_object_upl_request(vnode_object->control_handle, | |
694 | offset, size, &upl, NULL, NULL, request_flags); | |
695 | if (kr != KERN_SUCCESS) | |
696 | panic("vnode_pager_cluster_write: upl request failed\n"); | |
697 | ||
698 | vnode_pageout(vnode_object->vnode_handle, | |
699 | upl, (vm_offset_t)0, offset, size, upl_flags, &errno); | |
700 | ||
701 | if ( (upl_flags & UPL_KEEPCACHED) ) { | |
702 | if ( (*io_error = errno) ) | |
703 | break; | |
704 | } | |
705 | cnt -= size; | |
706 | offset += size; | |
707 | } | |
708 | if (resid_offset) | |
709 | *resid_offset = offset; | |
710 | ||
711 | } else { | |
712 | vm_object_offset_t vnode_size; | |
713 | vm_object_offset_t base_offset; | |
714 | vm_object_t object; | |
715 | vm_page_t target_page; | |
716 | int ticket; | |
717 | ||
718 | /* | |
719 | * this is the pageout path | |
720 | */ | |
721 | vnode_size = vnode_pager_get_filesize(vnode_object->vnode_handle); | |
722 | ||
723 | if (vnode_size > (offset + PAGE_SIZE)) { | |
724 | /* | |
725 | * preset the maximum size of the cluster | |
726 | * and put us on a nice cluster boundary... | |
727 | * and then clip the size to insure we | |
728 | * don't request past the end of the underlying file | |
729 | */ | |
730 | size = PAGE_SIZE * MAX_UPL_TRANSFER; | |
731 | base_offset = offset & ~((signed)(size - 1)); | |
732 | ||
733 | if ((base_offset + size) > vnode_size) | |
734 | size = round_page_32(((vm_size_t)(vnode_size - base_offset))); | |
735 | } else { | |
736 | /* | |
737 | * we've been requested to page out a page beyond the current | |
738 | * end of the 'file'... don't try to cluster in this case... | |
739 | * we still need to send this page through because it might | |
740 | * be marked precious and the underlying filesystem may need | |
741 | * to do something with it (besides page it out)... | |
742 | */ | |
743 | base_offset = offset; | |
744 | size = PAGE_SIZE; | |
745 | } | |
746 | object = memory_object_control_to_vm_object(vnode_object->control_handle); | |
747 | ||
748 | if (object == VM_OBJECT_NULL) | |
749 | panic("vnode_pager_cluster_write: NULL vm_object in control handle\n"); | |
750 | ||
751 | request_flags = UPL_NOBLOCK | UPL_FOR_PAGEOUT | UPL_CLEAN_IN_PLACE | | |
752 | UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | | |
753 | UPL_SET_INTERNAL | UPL_SET_LITE; | |
754 | ||
755 | vm_object_lock(object); | |
756 | ||
757 | if ((target_page = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { | |
758 | /* | |
759 | * only pick up pages whose ticket number matches | |
760 | * the ticket number of the page orginally targeted | |
761 | * for pageout | |
762 | */ | |
763 | ticket = target_page->page_ticket; | |
764 | ||
765 | request_flags |= ((ticket << UPL_PAGE_TICKET_SHIFT) & UPL_PAGE_TICKET_MASK); | |
766 | } | |
767 | vm_object_unlock(object); | |
768 | ||
769 | vm_object_upl_request(object, base_offset, size, | |
770 | &upl, NULL, NULL, request_flags); | |
771 | if (upl == NULL) | |
772 | panic("vnode_pager_cluster_write: upl request failed\n"); | |
773 | ||
774 | vnode_pageout(vnode_object->vnode_handle, | |
775 | upl, (vm_offset_t)0, upl->offset, upl->size, UPL_VNODE_PAGER, NULL); | |
776 | } | |
777 | } | |
778 | ||
779 | ||
780 | /* | |
781 | * | |
782 | */ | |
783 | kern_return_t | |
784 | vnode_pager_cluster_read( | |
785 | vnode_pager_t vnode_object, | |
786 | vm_object_offset_t offset, | |
787 | vm_size_t cnt) | |
788 | { | |
789 | int local_error = 0; | |
790 | int kret; | |
791 | ||
792 | assert(! (cnt & PAGE_MASK)); | |
793 | ||
794 | kret = vnode_pagein(vnode_object->vnode_handle, | |
795 | (upl_t) NULL, | |
796 | (vm_offset_t) NULL, | |
797 | offset, | |
798 | cnt, | |
799 | 0, | |
800 | &local_error); | |
801 | /* | |
802 | if(kret == PAGER_ABSENT) { | |
803 | Need to work out the defs here, 1 corresponds to PAGER_ABSENT | |
804 | defined in bsd/vm/vm_pager.h However, we should not be including | |
805 | that file here it is a layering violation. | |
806 | */ | |
807 | if (kret == 1) { | |
808 | int uplflags; | |
809 | upl_t upl = NULL; | |
810 | int count = 0; | |
811 | kern_return_t kr; | |
812 | ||
813 | uplflags = (UPL_NO_SYNC | | |
814 | UPL_CLEAN_IN_PLACE | | |
815 | UPL_SET_INTERNAL); | |
816 | count = 0; | |
817 | kr = memory_object_upl_request(vnode_object->control_handle, | |
818 | offset, cnt, | |
819 | &upl, NULL, &count, uplflags); | |
820 | if (kr == KERN_SUCCESS) { | |
821 | upl_abort(upl, 0); | |
822 | upl_deallocate(upl); | |
823 | } else { | |
824 | /* | |
825 | * We couldn't gather the page list, probably | |
826 | * because the memory object doesn't have a link | |
827 | * to a VM object anymore (forced unmount, for | |
828 | * example). Just return an error to the vm_fault() | |
829 | * path and let it handle it. | |
830 | */ | |
831 | } | |
832 | ||
833 | return KERN_FAILURE; | |
834 | } | |
835 | ||
836 | return KERN_SUCCESS; | |
837 | ||
838 | } | |
839 | ||
840 | ||
841 | /* | |
842 | * | |
843 | */ | |
844 | void | |
845 | vnode_pager_release_from_cache( | |
846 | int *cnt) | |
847 | { | |
848 | memory_object_free_from_cache( | |
849 | &realhost, &vnode_pager_workaround, cnt); | |
850 | } | |
851 | ||
852 | /* | |
853 | * | |
854 | */ | |
855 | vnode_pager_t | |
856 | vnode_object_create( | |
857 | struct vnode *vp) | |
858 | { | |
859 | register vnode_pager_t vnode_object; | |
860 | ||
861 | vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone); | |
862 | if (vnode_object == VNODE_PAGER_NULL) | |
863 | return(VNODE_PAGER_NULL); | |
864 | ||
865 | /* | |
866 | * The vm_map call takes both named entry ports and raw memory | |
867 | * objects in the same parameter. We need to make sure that | |
868 | * vm_map does not see this object as a named entry port. So, | |
869 | * we reserve the second word in the object for a fake ip_kotype | |
870 | * setting - that will tell vm_map to use it as a memory object. | |
871 | */ | |
872 | vnode_object->pager = &vnode_pager_workaround; | |
873 | vnode_object->pager_ikot = IKOT_MEMORY_OBJECT; | |
874 | vnode_object->ref_count = 1; | |
875 | vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL; | |
876 | vnode_object->vnode_handle = vp; | |
877 | ||
878 | return(vnode_object); | |
879 | } | |
880 | ||
881 | /* | |
882 | * | |
883 | */ | |
884 | vnode_pager_t | |
885 | vnode_pager_lookup( | |
886 | memory_object_t name) | |
887 | { | |
888 | vnode_pager_t vnode_object; | |
889 | ||
890 | vnode_object = (vnode_pager_t)name; | |
891 | assert(vnode_object->pager == &vnode_pager_workaround); | |
892 | return (vnode_object); | |
893 | } | |
894 |