]>
Commit | Line | Data |
---|---|---|
1c79356b | 1 | /* |
91447636 | 2 | * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved. |
1c79356b A |
3 | * |
4 | * @APPLE_LICENSE_HEADER_START@ | |
5 | * | |
37839358 A |
6 | * The contents of this file constitute Original Code as defined in and |
7 | * are subject to the Apple Public Source License Version 1.1 (the | |
8 | * "License"). You may not use this file except in compliance with the | |
9 | * License. Please obtain a copy of the License at | |
10 | * http://www.apple.com/publicsource and read it before using this file. | |
1c79356b | 11 | * |
37839358 A |
12 | * This Original Code and all software distributed under the License are |
13 | * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
1c79356b A |
14 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
15 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
37839358 A |
16 | * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the |
17 | * License for the specific language governing rights and limitations | |
18 | * under the License. | |
1c79356b A |
19 | * |
20 | * @APPLE_LICENSE_HEADER_END@ | |
21 | */ | |
22 | ||
23 | #include <sys/errno.h> | |
91447636 | 24 | |
1c79356b | 25 | #include <mach/mach_types.h> |
91447636 A |
26 | #include <mach/mach_traps.h> |
27 | #include <mach/host_priv.h> | |
1c79356b | 28 | #include <mach/kern_return.h> |
91447636 | 29 | #include <mach/memory_object_control.h> |
1c79356b A |
30 | #include <mach/memory_object_types.h> |
31 | #include <mach/port.h> | |
32 | #include <mach/policy.h> | |
91447636 A |
33 | #include <mach/upl.h> |
34 | #include <mach/thread_act.h> | |
35 | ||
36 | #include <kern/host.h> | |
37 | #include <kern/thread.h> | |
38 | ||
1c79356b A |
39 | #include <ipc/ipc_port.h> |
40 | #include <ipc/ipc_space.h> | |
1c79356b | 41 | |
0b4e3aa0 | 42 | #include <default_pager/default_pager_types.h> |
91447636 A |
43 | #include <default_pager/default_pager_object_server.h> |
44 | ||
45 | #include <vm/vm_map.h> | |
46 | #include <vm/vm_kern.h> | |
47 | #include <vm/vm_pageout.h> | |
48 | #include <vm/memory_object.h> | |
49 | #include <vm/vm_pageout.h> | |
50 | #include <vm/vm_protos.h> | |
1c79356b A |
51 | |
52 | /* BSD VM COMPONENT INTERFACES */ | |
53 | int | |
54 | get_map_nentries( | |
55 | vm_map_t); | |
56 | ||
57 | vm_offset_t | |
58 | get_map_start( | |
59 | vm_map_t); | |
60 | ||
61 | vm_offset_t | |
62 | get_map_end( | |
63 | vm_map_t); | |
64 | ||
65 | /* | |
66 | * | |
67 | */ | |
68 | int | |
69 | get_map_nentries( | |
70 | vm_map_t map) | |
71 | { | |
72 | return(map->hdr.nentries); | |
73 | } | |
74 | ||
91447636 A |
75 | mach_vm_offset_t |
76 | mach_get_vm_start(vm_map_t map) | |
77 | { | |
78 | return( vm_map_first_entry(map)->vme_start); | |
79 | } | |
80 | ||
81 | mach_vm_offset_t | |
82 | mach_get_vm_end(vm_map_t map) | |
83 | { | |
84 | return( vm_map_last_entry(map)->vme_end); | |
85 | } | |
86 | ||
1c79356b | 87 | /* |
91447636 A |
88 | * Legacy routines to get the start and end for a vm_map_t. They |
89 | * return them in the vm_offset_t format. So, they should only be | |
90 | * called on maps that are the same size as the kernel map for | |
91 | * accurate results. | |
1c79356b A |
92 | */ |
93 | vm_offset_t | |
91447636 | 94 | get_vm_start( |
1c79356b A |
95 | vm_map_t map) |
96 | { | |
91447636 | 97 | return(CAST_DOWN(vm_offset_t, vm_map_first_entry(map)->vme_start)); |
1c79356b A |
98 | } |
99 | ||
1c79356b | 100 | vm_offset_t |
91447636 | 101 | get_vm_end( |
1c79356b A |
102 | vm_map_t map) |
103 | { | |
91447636 | 104 | return(CAST_DOWN(vm_offset_t, vm_map_last_entry(map)->vme_end)); |
1c79356b A |
105 | } |
106 | ||
107 | /* | |
108 | * BSD VNODE PAGER | |
109 | */ | |
110 | ||
111 | /* until component support available */ | |
112 | int vnode_pager_workaround; | |
113 | ||
1c79356b | 114 | typedef struct vnode_pager { |
0b4e3aa0 A |
115 | int *pager; /* pager workaround pointer */ |
116 | unsigned int pager_ikot; /* JMM: fake ip_kotype() */ | |
117 | unsigned int ref_count; /* reference count */ | |
118 | memory_object_control_t control_handle; /* mem object control handle */ | |
91447636 | 119 | struct vnode *vnode_handle; /* vnode handle */ |
1c79356b A |
120 | } *vnode_pager_t; |
121 | ||
1c79356b A |
122 | |
123 | ipc_port_t | |
91447636 | 124 | trigger_name_to_port( /* forward */ |
1c79356b A |
125 | mach_port_t); |
126 | ||
1c79356b | 127 | kern_return_t |
91447636 | 128 | vnode_pager_cluster_read( /* forward */ |
1c79356b A |
129 | vnode_pager_t, |
130 | vm_object_offset_t, | |
131 | vm_size_t); | |
132 | ||
133 | void | |
91447636 | 134 | vnode_pager_cluster_write( /* forward */ |
1c79356b A |
135 | vnode_pager_t, |
136 | vm_object_offset_t, | |
91447636 A |
137 | vm_size_t, |
138 | vm_object_offset_t *, | |
139 | int *, | |
140 | int); | |
1c79356b | 141 | |
0b4e3aa0 | 142 | |
1c79356b | 143 | vnode_pager_t |
91447636 A |
144 | vnode_object_create( /* forward */ |
145 | struct vnode *); | |
1c79356b | 146 | |
1c79356b | 147 | vnode_pager_t |
91447636 | 148 | vnode_pager_lookup( /* forward */ |
0b4e3aa0 | 149 | memory_object_t); |
1c79356b | 150 | |
1c79356b A |
151 | zone_t vnode_pager_zone; |
152 | ||
153 | ||
154 | #define VNODE_PAGER_NULL ((vnode_pager_t) 0) | |
155 | ||
156 | /* TODO: Should be set dynamically by vnode_pager_init() */ | |
157 | #define CLUSTER_SHIFT 1 | |
158 | ||
159 | /* TODO: Should be set dynamically by vnode_pager_bootstrap() */ | |
160 | #define MAX_VNODE 10000 | |
161 | ||
162 | ||
163 | #if DEBUG | |
164 | int pagerdebug=0; | |
165 | ||
166 | #define PAGER_ALL 0xffffffff | |
167 | #define PAGER_INIT 0x00000001 | |
168 | #define PAGER_PAGEIN 0x00000002 | |
169 | ||
170 | #define PAGER_DEBUG(LEVEL, A) {if ((pagerdebug & LEVEL)==LEVEL){printf A;}} | |
171 | #else | |
172 | #define PAGER_DEBUG(LEVEL, A) | |
173 | #endif | |
174 | ||
175 | /* | |
176 | * Routine: macx_triggers | |
177 | * Function: | |
178 | * Syscall interface to set the call backs for low and | |
179 | * high water marks. | |
180 | */ | |
181 | int | |
182 | macx_triggers( | |
91447636 | 183 | struct macx_triggers_args *args) |
1c79356b | 184 | { |
91447636 A |
185 | int hi_water = args->hi_water; |
186 | int low_water = args->low_water; | |
187 | int flags = args->flags; | |
188 | mach_port_t trigger_name = args->alert_port; | |
1c79356b | 189 | kern_return_t kr; |
0b4e3aa0 | 190 | memory_object_default_t default_pager; |
1c79356b A |
191 | ipc_port_t trigger_port; |
192 | ||
0b4e3aa0 | 193 | default_pager = MEMORY_OBJECT_DEFAULT_NULL; |
1c79356b | 194 | kr = host_default_memory_manager(host_priv_self(), |
0b4e3aa0 | 195 | &default_pager, 0); |
1c79356b A |
196 | if(kr != KERN_SUCCESS) { |
197 | return EINVAL; | |
198 | } | |
91447636 A |
199 | |
200 | if ((flags & SWAP_ENCRYPT_ON) && | |
201 | (flags & SWAP_ENCRYPT_OFF)) { | |
202 | /* can't have it both ways */ | |
203 | return EINVAL; | |
204 | } | |
205 | ||
206 | if (flags & SWAP_ENCRYPT_ON) { | |
207 | /* ENCRYPTED SWAP: tell default_pager to encrypt */ | |
208 | default_pager_triggers(default_pager, | |
209 | 0, 0, | |
210 | SWAP_ENCRYPT_ON, | |
211 | IP_NULL); | |
212 | } else if (flags & SWAP_ENCRYPT_OFF) { | |
213 | /* ENCRYPTED SWAP: tell default_pager not to encrypt */ | |
214 | default_pager_triggers(default_pager, | |
215 | 0, 0, | |
216 | SWAP_ENCRYPT_OFF, | |
217 | IP_NULL); | |
218 | } | |
219 | ||
0b4e3aa0 A |
220 | if (flags & HI_WAT_ALERT) { |
221 | trigger_port = trigger_name_to_port(trigger_name); | |
222 | if(trigger_port == NULL) { | |
223 | return EINVAL; | |
224 | } | |
225 | /* trigger_port is locked and active */ | |
226 | ipc_port_make_send_locked(trigger_port); | |
227 | /* now unlocked */ | |
228 | default_pager_triggers(default_pager, | |
229 | hi_water, low_water, | |
230 | HI_WAT_ALERT, trigger_port); | |
231 | } | |
232 | ||
233 | if (flags & LO_WAT_ALERT) { | |
234 | trigger_port = trigger_name_to_port(trigger_name); | |
235 | if(trigger_port == NULL) { | |
236 | return EINVAL; | |
237 | } | |
238 | /* trigger_port is locked and active */ | |
239 | ipc_port_make_send_locked(trigger_port); | |
240 | /* and now its unlocked */ | |
241 | default_pager_triggers(default_pager, | |
242 | hi_water, low_water, | |
243 | LO_WAT_ALERT, trigger_port); | |
1c79356b | 244 | } |
1c79356b A |
245 | |
246 | /* | |
247 | * Set thread scheduling priority and policy for the current thread | |
248 | * it is assumed for the time being that the thread setting the alert | |
55e303ae A |
249 | * is the same one which will be servicing it. |
250 | * | |
251 | * XXX This does not belong in the kernel XXX | |
1c79356b A |
252 | */ |
253 | { | |
55e303ae A |
254 | thread_precedence_policy_data_t pre; |
255 | thread_extended_policy_data_t ext; | |
256 | ||
257 | ext.timeshare = FALSE; | |
258 | pre.importance = INT32_MAX; | |
259 | ||
91447636 A |
260 | thread_policy_set(current_thread(), |
261 | THREAD_EXTENDED_POLICY, | |
262 | (thread_policy_t)&ext, | |
263 | THREAD_EXTENDED_POLICY_COUNT); | |
55e303ae | 264 | |
91447636 A |
265 | thread_policy_set(current_thread(), |
266 | THREAD_PRECEDENCE_POLICY, | |
267 | (thread_policy_t)&pre, | |
268 | THREAD_PRECEDENCE_POLICY_COUNT); | |
1c79356b A |
269 | } |
270 | ||
91447636 A |
271 | current_thread()->options |= TH_OPT_VMPRIV; |
272 | ||
273 | return 0; | |
1c79356b A |
274 | } |
275 | ||
276 | /* | |
277 | * | |
278 | */ | |
279 | ipc_port_t | |
280 | trigger_name_to_port( | |
281 | mach_port_t trigger_name) | |
282 | { | |
283 | ipc_port_t trigger_port; | |
284 | ipc_space_t space; | |
285 | ||
286 | if (trigger_name == 0) | |
287 | return (NULL); | |
288 | ||
289 | space = current_space(); | |
290 | if(ipc_port_translate_receive(space, (mach_port_name_t)trigger_name, | |
291 | &trigger_port) != KERN_SUCCESS) | |
292 | return (NULL); | |
293 | return trigger_port; | |
294 | } | |
295 | ||
91447636 A |
296 | |
297 | extern int uiomove64(addr64_t, int, void *); | |
298 | #define MAX_RUN 32 | |
299 | ||
300 | int | |
301 | memory_object_control_uiomove( | |
302 | memory_object_control_t control, | |
303 | memory_object_offset_t offset, | |
304 | void * uio, | |
305 | int start_offset, | |
306 | int io_requested, | |
307 | int mark_dirty) | |
308 | { | |
309 | vm_object_t object; | |
310 | vm_page_t dst_page; | |
311 | int xsize; | |
312 | int retval = 0; | |
313 | int cur_run; | |
314 | int cur_needed; | |
315 | int i; | |
316 | vm_page_t page_run[MAX_RUN]; | |
317 | ||
318 | ||
319 | object = memory_object_control_to_vm_object(control); | |
320 | if (object == VM_OBJECT_NULL) { | |
321 | return (0); | |
322 | } | |
323 | assert(!object->internal); | |
324 | ||
325 | vm_object_lock(object); | |
326 | ||
327 | if (mark_dirty && object->copy != VM_OBJECT_NULL) { | |
328 | /* | |
329 | * We can't modify the pages without honoring | |
330 | * copy-on-write obligations first, so fall off | |
331 | * this optimized path and fall back to the regular | |
332 | * path. | |
333 | */ | |
334 | vm_object_unlock(object); | |
335 | return 0; | |
336 | } | |
337 | ||
338 | while (io_requested && retval == 0) { | |
339 | ||
340 | cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE; | |
341 | ||
342 | if (cur_needed > MAX_RUN) | |
343 | cur_needed = MAX_RUN; | |
344 | ||
345 | for (cur_run = 0; cur_run < cur_needed; ) { | |
346 | ||
347 | if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL) | |
348 | break; | |
349 | /* | |
350 | * Sync up on getting the busy bit | |
351 | */ | |
352 | if ((dst_page->busy || dst_page->cleaning)) { | |
353 | /* | |
354 | * someone else is playing with the page... if we've | |
355 | * already collected pages into this run, go ahead | |
356 | * and process now, we can't block on this | |
357 | * page while holding other pages in the BUSY state | |
358 | * otherwise we will wait | |
359 | */ | |
360 | if (cur_run) | |
361 | break; | |
362 | PAGE_SLEEP(object, dst_page, THREAD_UNINT); | |
363 | continue; | |
364 | } | |
365 | /* | |
366 | * this routine is only called when copying | |
367 | * to/from real files... no need to consider | |
368 | * encrypted swap pages | |
369 | */ | |
370 | assert(!dst_page->encrypted); | |
371 | ||
372 | if (mark_dirty) | |
373 | dst_page->dirty = TRUE; | |
374 | dst_page->busy = TRUE; | |
375 | ||
376 | page_run[cur_run++] = dst_page; | |
377 | ||
378 | offset += PAGE_SIZE_64; | |
379 | } | |
380 | if (cur_run == 0) | |
381 | /* | |
382 | * we hit a 'hole' in the cache | |
383 | * we bail at this point | |
384 | * we'll unlock the object below | |
385 | */ | |
386 | break; | |
387 | vm_object_unlock(object); | |
388 | ||
389 | for (i = 0; i < cur_run; i++) { | |
390 | ||
391 | dst_page = page_run[i]; | |
392 | ||
393 | if ((xsize = PAGE_SIZE - start_offset) > io_requested) | |
394 | xsize = io_requested; | |
395 | ||
396 | if ( (retval = uiomove64((addr64_t)(((addr64_t)(dst_page->phys_page) << 12) + start_offset), xsize, uio)) ) | |
397 | break; | |
398 | ||
399 | io_requested -= xsize; | |
400 | start_offset = 0; | |
401 | } | |
402 | vm_object_lock(object); | |
403 | ||
404 | for (i = 0; i < cur_run; i++) { | |
405 | dst_page = page_run[i]; | |
406 | ||
407 | PAGE_WAKEUP_DONE(dst_page); | |
408 | } | |
409 | } | |
410 | vm_object_unlock(object); | |
411 | ||
412 | return (retval); | |
413 | } | |
414 | ||
415 | ||
1c79356b A |
416 | /* |
417 | * | |
418 | */ | |
419 | void | |
420 | vnode_pager_bootstrap(void) | |
421 | { | |
422 | register vm_size_t size; | |
423 | ||
424 | size = (vm_size_t) sizeof(struct vnode_pager); | |
425 | vnode_pager_zone = zinit(size, (vm_size_t) MAX_VNODE*size, | |
426 | PAGE_SIZE, "vnode pager structures"); | |
1c79356b A |
427 | return; |
428 | } | |
429 | ||
430 | /* | |
431 | * | |
432 | */ | |
0b4e3aa0 | 433 | memory_object_t |
1c79356b | 434 | vnode_pager_setup( |
91447636 A |
435 | struct vnode *vp, |
436 | __unused memory_object_t pager) | |
1c79356b A |
437 | { |
438 | vnode_pager_t vnode_object; | |
1c79356b A |
439 | |
440 | vnode_object = vnode_object_create(vp); | |
441 | if (vnode_object == VNODE_PAGER_NULL) | |
442 | panic("vnode_pager_setup: vnode_object_create() failed"); | |
0b4e3aa0 | 443 | return((memory_object_t)vnode_object); |
1c79356b A |
444 | } |
445 | ||
446 | /* | |
447 | * | |
448 | */ | |
449 | kern_return_t | |
0b4e3aa0 A |
450 | vnode_pager_init(memory_object_t mem_obj, |
451 | memory_object_control_t control, | |
91447636 A |
452 | #if !DEBUG |
453 | __unused | |
454 | #endif | |
455 | vm_size_t pg_size) | |
1c79356b A |
456 | { |
457 | vnode_pager_t vnode_object; | |
458 | kern_return_t kr; | |
459 | memory_object_attr_info_data_t attributes; | |
1c79356b A |
460 | |
461 | ||
91447636 | 462 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_init: %p, %p, %x\n", mem_obj, control, pg_size)); |
1c79356b | 463 | |
0b4e3aa0 A |
464 | if (control == MEMORY_OBJECT_CONTROL_NULL) |
465 | return KERN_INVALID_ARGUMENT; | |
1c79356b | 466 | |
0b4e3aa0 | 467 | vnode_object = vnode_pager_lookup(mem_obj); |
1c79356b | 468 | |
0b4e3aa0 | 469 | memory_object_control_reference(control); |
91447636 | 470 | |
0b4e3aa0 | 471 | vnode_object->control_handle = control; |
1c79356b A |
472 | |
473 | attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; | |
474 | /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ | |
475 | attributes.cluster_size = (1 << (PAGE_SHIFT)); | |
476 | attributes.may_cache_object = TRUE; | |
477 | attributes.temporary = TRUE; | |
478 | ||
479 | kr = memory_object_change_attributes( | |
0b4e3aa0 | 480 | control, |
1c79356b A |
481 | MEMORY_OBJECT_ATTRIBUTE_INFO, |
482 | (memory_object_info_t) &attributes, | |
0b4e3aa0 | 483 | MEMORY_OBJECT_ATTR_INFO_COUNT); |
1c79356b A |
484 | if (kr != KERN_SUCCESS) |
485 | panic("vnode_pager_init: memory_object_change_attributes() failed"); | |
486 | ||
487 | return(KERN_SUCCESS); | |
488 | } | |
489 | ||
490 | /* | |
491 | * | |
492 | */ | |
493 | kern_return_t | |
494 | vnode_pager_data_return( | |
0b4e3aa0 A |
495 | memory_object_t mem_obj, |
496 | memory_object_offset_t offset, | |
1c79356b | 497 | vm_size_t data_cnt, |
91447636 A |
498 | memory_object_offset_t *resid_offset, |
499 | int *io_error, | |
500 | __unused boolean_t dirty, | |
501 | __unused boolean_t kernel_copy, | |
502 | int upl_flags) | |
1c79356b A |
503 | { |
504 | register vnode_pager_t vnode_object; | |
505 | ||
0b4e3aa0 | 506 | vnode_object = vnode_pager_lookup(mem_obj); |
1c79356b | 507 | |
91447636 | 508 | vnode_pager_cluster_write(vnode_object, offset, data_cnt, resid_offset, io_error, upl_flags); |
1c79356b A |
509 | |
510 | return KERN_SUCCESS; | |
511 | } | |
512 | ||
0b4e3aa0 A |
513 | kern_return_t |
514 | vnode_pager_data_initialize( | |
91447636 A |
515 | __unused memory_object_t mem_obj, |
516 | __unused memory_object_offset_t offset, | |
517 | __unused vm_size_t data_cnt) | |
0b4e3aa0 | 518 | { |
91447636 | 519 | panic("vnode_pager_data_initialize"); |
0b4e3aa0 A |
520 | return KERN_FAILURE; |
521 | } | |
522 | ||
523 | kern_return_t | |
524 | vnode_pager_data_unlock( | |
91447636 A |
525 | __unused memory_object_t mem_obj, |
526 | __unused memory_object_offset_t offset, | |
527 | __unused vm_size_t size, | |
528 | __unused vm_prot_t desired_access) | |
0b4e3aa0 A |
529 | { |
530 | return KERN_FAILURE; | |
531 | } | |
532 | ||
533 | kern_return_t | |
534 | vnode_pager_get_object_size( | |
535 | memory_object_t mem_obj, | |
536 | memory_object_offset_t *length) | |
537 | { | |
538 | vnode_pager_t vnode_object; | |
539 | ||
540 | vnode_object = vnode_pager_lookup(mem_obj); | |
541 | ||
542 | *length = vnode_pager_get_filesize(vnode_object->vnode_handle); | |
543 | return KERN_SUCCESS; | |
544 | } | |
545 | ||
1c79356b A |
546 | /* |
547 | * | |
548 | */ | |
549 | kern_return_t | |
550 | vnode_pager_data_request( | |
0b4e3aa0 A |
551 | memory_object_t mem_obj, |
552 | memory_object_offset_t offset, | |
1c79356b | 553 | vm_size_t length, |
91447636 A |
554 | #if !DEBUG |
555 | __unused | |
556 | #endif | |
557 | vm_prot_t protection_required) | |
1c79356b A |
558 | { |
559 | register vnode_pager_t vnode_object; | |
560 | ||
0b4e3aa0 | 561 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required)); |
1c79356b | 562 | |
0b4e3aa0 | 563 | vnode_object = vnode_pager_lookup(mem_obj); |
1c79356b | 564 | |
0b4e3aa0 | 565 | PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object)); |
1c79356b | 566 | |
91447636 | 567 | return vnode_pager_cluster_read(vnode_object, offset, length); |
1c79356b A |
568 | } |
569 | ||
570 | /* | |
571 | * | |
572 | */ | |
573 | void | |
0b4e3aa0 A |
574 | vnode_pager_reference( |
575 | memory_object_t mem_obj) | |
576 | { | |
1c79356b | 577 | register vnode_pager_t vnode_object; |
9bccf70c | 578 | unsigned int new_ref_count; |
1c79356b | 579 | |
0b4e3aa0 | 580 | vnode_object = vnode_pager_lookup(mem_obj); |
9bccf70c A |
581 | new_ref_count = hw_atomic_add(&vnode_object->ref_count, 1); |
582 | assert(new_ref_count > 1); | |
0b4e3aa0 | 583 | } |
1c79356b | 584 | |
0b4e3aa0 A |
585 | /* |
586 | * | |
587 | */ | |
588 | void | |
589 | vnode_pager_deallocate( | |
590 | memory_object_t mem_obj) | |
591 | { | |
592 | register vnode_pager_t vnode_object; | |
1c79356b | 593 | |
0b4e3aa0 | 594 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj)); |
1c79356b | 595 | |
0b4e3aa0 | 596 | vnode_object = vnode_pager_lookup(mem_obj); |
1c79356b | 597 | |
9bccf70c | 598 | if (hw_atomic_sub(&vnode_object->ref_count, 1) == 0) { |
91447636 | 599 | if (vnode_object->vnode_handle != NULL) { |
0b4e3aa0 A |
600 | vnode_pager_vrele(vnode_object->vnode_handle); |
601 | } | |
91447636 | 602 | zfree(vnode_pager_zone, vnode_object); |
0b4e3aa0 | 603 | } |
1c79356b A |
604 | return; |
605 | } | |
606 | ||
607 | /* | |
608 | * | |
609 | */ | |
610 | kern_return_t | |
611 | vnode_pager_terminate( | |
91447636 A |
612 | #if !DEBUG |
613 | __unused | |
614 | #endif | |
0b4e3aa0 | 615 | memory_object_t mem_obj) |
1c79356b | 616 | { |
0b4e3aa0 | 617 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj)); |
1c79356b | 618 | |
0b4e3aa0 A |
619 | return(KERN_SUCCESS); |
620 | } | |
1c79356b | 621 | |
0b4e3aa0 A |
622 | /* |
623 | * | |
624 | */ | |
625 | kern_return_t | |
626 | vnode_pager_synchronize( | |
627 | memory_object_t mem_obj, | |
628 | memory_object_offset_t offset, | |
629 | vm_size_t length, | |
91447636 | 630 | __unused vm_sync_t sync_flags) |
0b4e3aa0 A |
631 | { |
632 | register vnode_pager_t vnode_object; | |
1c79356b | 633 | |
0b4e3aa0 | 634 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj)); |
1c79356b | 635 | |
0b4e3aa0 | 636 | vnode_object = vnode_pager_lookup(mem_obj); |
1c79356b | 637 | |
0b4e3aa0 | 638 | memory_object_synchronize_completed(vnode_object->control_handle, offset, length); |
1c79356b | 639 | |
0b4e3aa0 | 640 | return (KERN_SUCCESS); |
1c79356b A |
641 | } |
642 | ||
643 | /* | |
644 | * | |
645 | */ | |
646 | kern_return_t | |
0b4e3aa0 A |
647 | vnode_pager_unmap( |
648 | memory_object_t mem_obj) | |
1c79356b | 649 | { |
0b4e3aa0 | 650 | register vnode_pager_t vnode_object; |
1c79356b | 651 | |
0b4e3aa0 A |
652 | PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj)); |
653 | ||
654 | vnode_object = vnode_pager_lookup(mem_obj); | |
655 | ||
656 | ubc_unmap(vnode_object->vnode_handle); | |
657 | return KERN_SUCCESS; | |
1c79356b A |
658 | } |
659 | ||
0b4e3aa0 | 660 | |
1c79356b A |
661 | /* |
662 | * | |
663 | */ | |
664 | void | |
665 | vnode_pager_cluster_write( | |
666 | vnode_pager_t vnode_object, | |
667 | vm_object_offset_t offset, | |
91447636 A |
668 | vm_size_t cnt, |
669 | vm_object_offset_t * resid_offset, | |
670 | int * io_error, | |
671 | int upl_flags) | |
1c79356b | 672 | { |
91447636 A |
673 | vm_size_t size; |
674 | upl_t upl = NULL; | |
675 | int request_flags; | |
676 | int errno; | |
1c79356b | 677 | |
91447636 | 678 | if (upl_flags & UPL_MSYNC) { |
1c79356b | 679 | |
91447636 A |
680 | upl_flags |= UPL_VNODE_PAGER; |
681 | ||
682 | if ( (upl_flags & UPL_IOSYNC) && io_error) | |
683 | upl_flags |= UPL_KEEPCACHED; | |
684 | ||
685 | while (cnt) { | |
686 | kern_return_t kr; | |
687 | ||
688 | size = (cnt < (PAGE_SIZE * MAX_UPL_TRANSFER)) ? cnt : (PAGE_SIZE * MAX_UPL_TRANSFER); /* effective max */ | |
0b4e3aa0 | 689 | |
91447636 A |
690 | request_flags = UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | UPL_CLEAN_IN_PLACE | |
691 | UPL_SET_INTERNAL | UPL_SET_LITE; | |
692 | ||
693 | kr = memory_object_upl_request(vnode_object->control_handle, | |
694 | offset, size, &upl, NULL, NULL, request_flags); | |
695 | if (kr != KERN_SUCCESS) | |
696 | panic("vnode_pager_cluster_write: upl request failed\n"); | |
697 | ||
698 | vnode_pageout(vnode_object->vnode_handle, | |
699 | upl, (vm_offset_t)0, offset, size, upl_flags, &errno); | |
700 | ||
701 | if ( (upl_flags & UPL_KEEPCACHED) ) { | |
702 | if ( (*io_error = errno) ) | |
703 | break; | |
704 | } | |
705 | cnt -= size; | |
706 | offset += size; | |
707 | } | |
708 | if (resid_offset) | |
709 | *resid_offset = offset; | |
710 | ||
711 | } else { | |
712 | vm_object_offset_t vnode_size; | |
713 | vm_object_offset_t base_offset; | |
714 | vm_object_t object; | |
715 | vm_page_t target_page; | |
716 | int ticket; | |
717 | ||
718 | /* | |
719 | * this is the pageout path | |
720 | */ | |
721 | vnode_size = vnode_pager_get_filesize(vnode_object->vnode_handle); | |
722 | ||
723 | if (vnode_size > (offset + PAGE_SIZE)) { | |
724 | /* | |
725 | * preset the maximum size of the cluster | |
726 | * and put us on a nice cluster boundary... | |
727 | * and then clip the size to insure we | |
728 | * don't request past the end of the underlying file | |
729 | */ | |
730 | size = PAGE_SIZE * MAX_UPL_TRANSFER; | |
731 | base_offset = offset & ~((signed)(size - 1)); | |
732 | ||
733 | if ((base_offset + size) > vnode_size) | |
734 | size = round_page_32(((vm_size_t)(vnode_size - base_offset))); | |
735 | } else { | |
736 | /* | |
737 | * we've been requested to page out a page beyond the current | |
738 | * end of the 'file'... don't try to cluster in this case... | |
739 | * we still need to send this page through because it might | |
740 | * be marked precious and the underlying filesystem may need | |
741 | * to do something with it (besides page it out)... | |
742 | */ | |
743 | base_offset = offset; | |
744 | size = PAGE_SIZE; | |
0b4e3aa0 | 745 | } |
91447636 A |
746 | object = memory_object_control_to_vm_object(vnode_object->control_handle); |
747 | ||
748 | if (object == VM_OBJECT_NULL) | |
749 | panic("vnode_pager_cluster_write: NULL vm_object in control handle\n"); | |
750 | ||
751 | request_flags = UPL_NOBLOCK | UPL_FOR_PAGEOUT | UPL_CLEAN_IN_PLACE | | |
752 | UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | | |
753 | UPL_SET_INTERNAL | UPL_SET_LITE; | |
754 | ||
755 | vm_object_lock(object); | |
756 | ||
757 | if ((target_page = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { | |
758 | /* | |
759 | * only pick up pages whose ticket number matches | |
760 | * the ticket number of the page orginally targeted | |
761 | * for pageout | |
762 | */ | |
763 | ticket = target_page->page_ticket; | |
1c79356b | 764 | |
91447636 | 765 | request_flags |= ((ticket << UPL_PAGE_TICKET_SHIFT) & UPL_PAGE_TICKET_MASK); |
1c79356b | 766 | } |
91447636 A |
767 | vm_object_unlock(object); |
768 | ||
769 | vm_object_upl_request(object, base_offset, size, | |
770 | &upl, NULL, NULL, request_flags); | |
771 | if (upl == NULL) | |
772 | panic("vnode_pager_cluster_write: upl request failed\n"); | |
773 | ||
774 | vnode_pageout(vnode_object->vnode_handle, | |
775 | upl, (vm_offset_t)0, upl->offset, upl->size, UPL_VNODE_PAGER, NULL); | |
1c79356b | 776 | } |
1c79356b A |
777 | } |
778 | ||
779 | ||
780 | /* | |
781 | * | |
782 | */ | |
783 | kern_return_t | |
784 | vnode_pager_cluster_read( | |
785 | vnode_pager_t vnode_object, | |
786 | vm_object_offset_t offset, | |
787 | vm_size_t cnt) | |
788 | { | |
1c79356b A |
789 | int local_error = 0; |
790 | int kret; | |
1c79356b | 791 | |
91447636 | 792 | assert(! (cnt & PAGE_MASK)); |
1c79356b | 793 | |
91447636 A |
794 | kret = vnode_pagein(vnode_object->vnode_handle, |
795 | (upl_t) NULL, | |
796 | (vm_offset_t) NULL, | |
797 | offset, | |
798 | cnt, | |
799 | 0, | |
800 | &local_error); | |
0b4e3aa0 A |
801 | /* |
802 | if(kret == PAGER_ABSENT) { | |
803 | Need to work out the defs here, 1 corresponds to PAGER_ABSENT | |
804 | defined in bsd/vm/vm_pager.h However, we should not be including | |
805 | that file here it is a layering violation. | |
806 | */ | |
91447636 A |
807 | if (kret == 1) { |
808 | int uplflags; | |
809 | upl_t upl = NULL; | |
810 | int count = 0; | |
811 | kern_return_t kr; | |
812 | ||
813 | uplflags = (UPL_NO_SYNC | | |
814 | UPL_CLEAN_IN_PLACE | | |
815 | UPL_SET_INTERNAL); | |
816 | count = 0; | |
817 | kr = memory_object_upl_request(vnode_object->control_handle, | |
818 | offset, cnt, | |
819 | &upl, NULL, &count, uplflags); | |
820 | if (kr == KERN_SUCCESS) { | |
0b4e3aa0 A |
821 | upl_abort(upl, 0); |
822 | upl_deallocate(upl); | |
91447636 A |
823 | } else { |
824 | /* | |
825 | * We couldn't gather the page list, probably | |
826 | * because the memory object doesn't have a link | |
827 | * to a VM object anymore (forced unmount, for | |
828 | * example). Just return an error to the vm_fault() | |
829 | * path and let it handle it. | |
830 | */ | |
831 | } | |
0b4e3aa0 | 832 | |
91447636 | 833 | return KERN_FAILURE; |
1c79356b | 834 | } |
0b4e3aa0 | 835 | |
91447636 | 836 | return KERN_SUCCESS; |
1c79356b A |
837 | |
838 | } | |
839 | ||
840 | ||
841 | /* | |
842 | * | |
843 | */ | |
844 | void | |
845 | vnode_pager_release_from_cache( | |
846 | int *cnt) | |
847 | { | |
848 | memory_object_free_from_cache( | |
0b4e3aa0 | 849 | &realhost, &vnode_pager_workaround, cnt); |
1c79356b A |
850 | } |
851 | ||
852 | /* | |
853 | * | |
854 | */ | |
855 | vnode_pager_t | |
856 | vnode_object_create( | |
91447636 | 857 | struct vnode *vp) |
1c79356b A |
858 | { |
859 | register vnode_pager_t vnode_object; | |
860 | ||
861 | vnode_object = (struct vnode_pager *) zalloc(vnode_pager_zone); | |
862 | if (vnode_object == VNODE_PAGER_NULL) | |
863 | return(VNODE_PAGER_NULL); | |
1c79356b | 864 | |
1c79356b | 865 | /* |
0b4e3aa0 A |
866 | * The vm_map call takes both named entry ports and raw memory |
867 | * objects in the same parameter. We need to make sure that | |
868 | * vm_map does not see this object as a named entry port. So, | |
869 | * we reserve the second word in the object for a fake ip_kotype | |
870 | * setting - that will tell vm_map to use it as a memory object. | |
1c79356b | 871 | */ |
0b4e3aa0 A |
872 | vnode_object->pager = &vnode_pager_workaround; |
873 | vnode_object->pager_ikot = IKOT_MEMORY_OBJECT; | |
874 | vnode_object->ref_count = 1; | |
875 | vnode_object->control_handle = MEMORY_OBJECT_CONTROL_NULL; | |
876 | vnode_object->vnode_handle = vp; | |
877 | ||
878 | return(vnode_object); | |
1c79356b A |
879 | } |
880 | ||
881 | /* | |
882 | * | |
883 | */ | |
884 | vnode_pager_t | |
0b4e3aa0 A |
885 | vnode_pager_lookup( |
886 | memory_object_t name) | |
1c79356b | 887 | { |
0b4e3aa0 | 888 | vnode_pager_t vnode_object; |
1c79356b | 889 | |
0b4e3aa0 A |
890 | vnode_object = (vnode_pager_t)name; |
891 | assert(vnode_object->pager == &vnode_pager_workaround); | |
892 | return (vnode_object); | |
1c79356b | 893 | } |
0b4e3aa0 | 894 |