| 1 | /* |
| 2 | * Copyright (c) 2008 Apple Computer, Inc. All rights reserved. |
| 3 | * |
| 4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
| 5 | * |
| 6 | * This file contains Original Code and/or Modifications of Original Code |
| 7 | * as defined in and that are subject to the Apple Public Source License |
| 8 | * Version 2.0 (the 'License'). You may not use this file except in |
| 9 | * compliance with the License. The rights granted to you under the License |
| 10 | * may not be used to create, or enable the creation or redistribution of, |
| 11 | * unlawful or unlicensed copies of an Apple operating system, or to |
| 12 | * circumvent, violate, or enable the circumvention or violation of, any |
| 13 | * terms of an Apple operating system software license agreement. |
| 14 | * |
| 15 | * Please obtain a copy of the License at |
| 16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
| 17 | * |
| 18 | * The Original Code and all software distributed under the License are |
| 19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
| 20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
| 21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
| 22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
| 23 | * Please see the License for the specific language governing rights and |
| 24 | * limitations under the License. |
| 25 | * |
| 26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
| 27 | */ |
| 28 | |
| 29 | #include <mach/kern_return.h> |
| 30 | #include <mach/memory_object_control.h> |
| 31 | #include <mach/upl.h> |
| 32 | |
| 33 | #include <kern/ipc_kobject.h> |
| 34 | #include <kern/kalloc.h> |
| 35 | #include <kern/queue.h> |
| 36 | |
| 37 | #include <vm/vm_kern.h> |
| 38 | #include <vm/vm_map.h> |
| 39 | #include <vm/vm_pageout.h> |
| 40 | #include <vm/vm_protos.h> |
| 41 | |
| 42 | |
| 43 | /* |
| 44 | * APPLE SWAPFILE MEMORY PAGER |
| 45 | * |
| 46 | * This external memory manager (EMM) handles mappings of the swap files. |
| 47 | * Swap files are not regular files and are used solely to store contents of |
| 48 | * anonymous memory mappings while not resident in memory. |
| 49 | * There's no valid reason to map a swap file. This just puts extra burden |
| 50 | * on the system, is potentially a security issue and is not reliable since |
| 51 | * the contents can change at any time with pageout operations. |
| 52 | * Here are some of the issues with mapping a swap file. |
| 53 | * * PERFORMANCE: |
| 54 | * Each page in the swap file belong to an anonymous memory object. Mapping |
| 55 | * the swap file makes those pages also accessible via a vnode memory |
| 56 | * object and each page can now be resident twice. |
| 57 | * * SECURITY: |
| 58 | * Mapping a swap file allows access to other processes' memory. Swap files |
| 59 | * are only accessible by the "root" super-user, who can already access any |
| 60 | * process's memory, so this is not a real issue but if permissions on the |
| 61 | * swap file got changed, it could become one. |
| 62 | * Swap files are not "zero-filled" on creation, so until their contents are |
| 63 | * overwritten with pageout operations, they still contain whatever was on |
| 64 | * the disk blocks they were allocated. The "super-user" could see the |
| 65 | * contents of free blocks anyway, so this is not a new security issue but |
| 66 | * it may be perceive as one. |
| 67 | * |
| 68 | * We can't legitimately prevent a user process with appropriate privileges |
| 69 | * from mapping a swap file, but we can prevent it from accessing its actual |
| 70 | * contents. |
| 71 | * This pager mostly handles page-in request (from memory_object_data_request()) |
| 72 | * for swap file mappings and just returns bogus data. |
| 73 | * Pageouts are not handled, so mmap() has to make sure it does not allow |
| 74 | * writable (i.e. MAP_SHARED and PROT_WRITE) mappings of swap files. |
| 75 | */ |
| 76 | |
| 77 | /* forward declarations */ |
| 78 | void swapfile_pager_reference(memory_object_t mem_obj); |
| 79 | void swapfile_pager_deallocate(memory_object_t mem_obj); |
| 80 | kern_return_t swapfile_pager_init(memory_object_t mem_obj, |
| 81 | memory_object_control_t control, |
| 82 | memory_object_cluster_size_t pg_size); |
| 83 | kern_return_t swapfile_pager_terminate(memory_object_t mem_obj); |
| 84 | kern_return_t swapfile_pager_data_request(memory_object_t mem_obj, |
| 85 | memory_object_offset_t offset, |
| 86 | memory_object_cluster_size_t length, |
| 87 | vm_prot_t protection_required, |
| 88 | memory_object_fault_info_t fault_info); |
| 89 | kern_return_t swapfile_pager_data_return(memory_object_t mem_obj, |
| 90 | memory_object_offset_t offset, |
| 91 | memory_object_cluster_size_t data_cnt, |
| 92 | memory_object_offset_t *resid_offset, |
| 93 | int *io_error, |
| 94 | boolean_t dirty, |
| 95 | boolean_t kernel_copy, |
| 96 | int upl_flags); |
| 97 | kern_return_t swapfile_pager_data_initialize(memory_object_t mem_obj, |
| 98 | memory_object_offset_t offset, |
| 99 | memory_object_cluster_size_t data_cnt); |
| 100 | kern_return_t swapfile_pager_data_unlock(memory_object_t mem_obj, |
| 101 | memory_object_offset_t offset, |
| 102 | memory_object_size_t size, |
| 103 | vm_prot_t desired_access); |
| 104 | kern_return_t swapfile_pager_synchronize(memory_object_t mem_obj, |
| 105 | memory_object_offset_t offset, |
| 106 | memory_object_size_t length, |
| 107 | vm_sync_t sync_flags); |
| 108 | kern_return_t swapfile_pager_map(memory_object_t mem_obj, |
| 109 | vm_prot_t prot); |
| 110 | kern_return_t swapfile_pager_last_unmap(memory_object_t mem_obj); |
| 111 | |
| 112 | /* |
| 113 | * Vector of VM operations for this EMM. |
| 114 | * These routines are invoked by VM via the memory_object_*() interfaces. |
| 115 | */ |
| 116 | const struct memory_object_pager_ops swapfile_pager_ops = { |
| 117 | swapfile_pager_reference, |
| 118 | swapfile_pager_deallocate, |
| 119 | swapfile_pager_init, |
| 120 | swapfile_pager_terminate, |
| 121 | swapfile_pager_data_request, |
| 122 | swapfile_pager_data_return, |
| 123 | swapfile_pager_data_initialize, |
| 124 | swapfile_pager_data_unlock, |
| 125 | swapfile_pager_synchronize, |
| 126 | swapfile_pager_map, |
| 127 | swapfile_pager_last_unmap, |
| 128 | NULL, /* data_reclaim */ |
| 129 | "swapfile pager" |
| 130 | }; |
| 131 | |
| 132 | /* |
| 133 | * The "swapfile_pager" describes a memory object backed by |
| 134 | * the "swapfile" EMM. |
| 135 | */ |
| 136 | typedef struct swapfile_pager { |
| 137 | /* mandatory generic header */ |
| 138 | struct memory_object swp_pgr_hdr; |
| 139 | |
| 140 | /* pager-specific data */ |
| 141 | queue_chain_t pager_queue; /* next & prev pagers */ |
| 142 | unsigned int ref_count; /* reference count */ |
| 143 | boolean_t is_ready; /* is this pager ready ? */ |
| 144 | boolean_t is_mapped; /* is this pager mapped ? */ |
| 145 | struct vnode *swapfile_vnode;/* the swapfile's vnode */ |
| 146 | } *swapfile_pager_t; |
| 147 | #define SWAPFILE_PAGER_NULL ((swapfile_pager_t) NULL) |
| 148 | |
| 149 | /* |
| 150 | * List of memory objects managed by this EMM. |
| 151 | * The list is protected by the "swapfile_pager_lock" lock. |
| 152 | */ |
| 153 | int swapfile_pager_count = 0; /* number of pagers */ |
| 154 | queue_head_t swapfile_pager_queue; |
| 155 | decl_lck_mtx_data(,swapfile_pager_lock) |
| 156 | |
| 157 | /* |
| 158 | * Statistics & counters. |
| 159 | */ |
| 160 | int swapfile_pager_count_max = 0; |
| 161 | |
| 162 | |
| 163 | lck_grp_t swapfile_pager_lck_grp; |
| 164 | lck_grp_attr_t swapfile_pager_lck_grp_attr; |
| 165 | lck_attr_t swapfile_pager_lck_attr; |
| 166 | |
| 167 | |
| 168 | /* internal prototypes */ |
| 169 | swapfile_pager_t swapfile_pager_create(struct vnode *vp); |
| 170 | swapfile_pager_t swapfile_pager_lookup(memory_object_t mem_obj); |
| 171 | void swapfile_pager_dequeue(swapfile_pager_t pager); |
| 172 | void swapfile_pager_deallocate_internal(swapfile_pager_t pager, |
| 173 | boolean_t locked); |
| 174 | void swapfile_pager_terminate_internal(swapfile_pager_t pager); |
| 175 | |
| 176 | |
| 177 | #if DEBUG |
| 178 | int swapfile_pagerdebug = 0; |
| 179 | #define PAGER_ALL 0xffffffff |
| 180 | #define PAGER_INIT 0x00000001 |
| 181 | #define PAGER_PAGEIN 0x00000002 |
| 182 | |
| 183 | #define PAGER_DEBUG(LEVEL, A) \ |
| 184 | MACRO_BEGIN \ |
| 185 | if ((swapfile_pagerdebug & LEVEL)==LEVEL) { \ |
| 186 | printf A; \ |
| 187 | } \ |
| 188 | MACRO_END |
| 189 | #else |
| 190 | #define PAGER_DEBUG(LEVEL, A) |
| 191 | #endif |
| 192 | |
| 193 | |
| 194 | void |
| 195 | swapfile_pager_bootstrap(void) |
| 196 | { |
| 197 | lck_grp_attr_setdefault(&swapfile_pager_lck_grp_attr); |
| 198 | lck_grp_init(&swapfile_pager_lck_grp, "swapfile pager", &swapfile_pager_lck_grp_attr); |
| 199 | lck_attr_setdefault(&swapfile_pager_lck_attr); |
| 200 | lck_mtx_init(&swapfile_pager_lock, &swapfile_pager_lck_grp, &swapfile_pager_lck_attr); |
| 201 | queue_init(&swapfile_pager_queue); |
| 202 | } |
| 203 | |
| 204 | /* |
| 205 | * swapfile_pager_init() |
| 206 | * |
| 207 | * Initialize the memory object and makes it ready to be used and mapped. |
| 208 | */ |
| 209 | kern_return_t |
| 210 | swapfile_pager_init( |
| 211 | memory_object_t mem_obj, |
| 212 | memory_object_control_t control, |
| 213 | #if !DEBUG |
| 214 | __unused |
| 215 | #endif |
| 216 | memory_object_cluster_size_t pg_size) |
| 217 | { |
| 218 | swapfile_pager_t pager; |
| 219 | kern_return_t kr; |
| 220 | memory_object_attr_info_data_t attributes; |
| 221 | |
| 222 | PAGER_DEBUG(PAGER_ALL, |
| 223 | ("swapfile_pager_init: %p, %p, %x\n", |
| 224 | mem_obj, control, pg_size)); |
| 225 | |
| 226 | if (control == MEMORY_OBJECT_CONTROL_NULL) |
| 227 | return KERN_INVALID_ARGUMENT; |
| 228 | |
| 229 | pager = swapfile_pager_lookup(mem_obj); |
| 230 | |
| 231 | memory_object_control_reference(control); |
| 232 | |
| 233 | pager->swp_pgr_hdr.mo_control = control; |
| 234 | |
| 235 | attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; |
| 236 | attributes.cluster_size = (1 << (PAGE_SHIFT)); |
| 237 | attributes.may_cache_object = FALSE; |
| 238 | attributes.temporary = TRUE; |
| 239 | |
| 240 | kr = memory_object_change_attributes( |
| 241 | control, |
| 242 | MEMORY_OBJECT_ATTRIBUTE_INFO, |
| 243 | (memory_object_info_t) &attributes, |
| 244 | MEMORY_OBJECT_ATTR_INFO_COUNT); |
| 245 | if (kr != KERN_SUCCESS) |
| 246 | panic("swapfile_pager_init: " |
| 247 | "memory_object_change_attributes() failed"); |
| 248 | |
| 249 | return KERN_SUCCESS; |
| 250 | } |
| 251 | |
| 252 | /* |
| 253 | * swapfile_data_return() |
| 254 | * |
| 255 | * Handles page-out requests from VM. This should never happen since |
| 256 | * the pages provided by this EMM are not supposed to be dirty or dirtied |
| 257 | * and VM should simply discard the contents and reclaim the pages if it |
| 258 | * needs to. |
| 259 | */ |
| 260 | kern_return_t |
| 261 | swapfile_pager_data_return( |
| 262 | __unused memory_object_t mem_obj, |
| 263 | __unused memory_object_offset_t offset, |
| 264 | __unused memory_object_cluster_size_t data_cnt, |
| 265 | __unused memory_object_offset_t *resid_offset, |
| 266 | __unused int *io_error, |
| 267 | __unused boolean_t dirty, |
| 268 | __unused boolean_t kernel_copy, |
| 269 | __unused int upl_flags) |
| 270 | { |
| 271 | panic("swapfile_pager_data_return: should never get called"); |
| 272 | return KERN_FAILURE; |
| 273 | } |
| 274 | |
| 275 | kern_return_t |
| 276 | swapfile_pager_data_initialize( |
| 277 | __unused memory_object_t mem_obj, |
| 278 | __unused memory_object_offset_t offset, |
| 279 | __unused memory_object_cluster_size_t data_cnt) |
| 280 | { |
| 281 | panic("swapfile_pager_data_initialize: should never get called"); |
| 282 | return KERN_FAILURE; |
| 283 | } |
| 284 | |
| 285 | kern_return_t |
| 286 | swapfile_pager_data_unlock( |
| 287 | __unused memory_object_t mem_obj, |
| 288 | __unused memory_object_offset_t offset, |
| 289 | __unused memory_object_size_t size, |
| 290 | __unused vm_prot_t desired_access) |
| 291 | { |
| 292 | return KERN_FAILURE; |
| 293 | } |
| 294 | |
| 295 | /* |
| 296 | * swapfile_pager_data_request() |
| 297 | * |
| 298 | * Handles page-in requests from VM. |
| 299 | */ |
| 300 | kern_return_t |
| 301 | swapfile_pager_data_request( |
| 302 | memory_object_t mem_obj, |
| 303 | memory_object_offset_t offset, |
| 304 | memory_object_cluster_size_t length, |
| 305 | #if !DEBUG |
| 306 | __unused |
| 307 | #endif |
| 308 | vm_prot_t protection_required, |
| 309 | __unused memory_object_fault_info_t mo_fault_info) |
| 310 | { |
| 311 | swapfile_pager_t pager; |
| 312 | memory_object_control_t mo_control; |
| 313 | upl_t upl; |
| 314 | int upl_flags; |
| 315 | upl_size_t upl_size; |
| 316 | upl_page_info_t *upl_pl = NULL; |
| 317 | unsigned int pl_count; |
| 318 | vm_object_t dst_object; |
| 319 | kern_return_t kr, retval; |
| 320 | vm_map_offset_t kernel_mapping; |
| 321 | vm_offset_t dst_vaddr; |
| 322 | char *dst_ptr; |
| 323 | vm_offset_t cur_offset; |
| 324 | vm_map_entry_t map_entry; |
| 325 | |
| 326 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required)); |
| 327 | |
| 328 | kernel_mapping = 0; |
| 329 | upl = NULL; |
| 330 | upl_pl = NULL; |
| 331 | |
| 332 | pager = swapfile_pager_lookup(mem_obj); |
| 333 | assert(pager->is_ready); |
| 334 | assert(pager->ref_count > 1); /* pager is alive and mapped */ |
| 335 | |
| 336 | PAGER_DEBUG(PAGER_PAGEIN, ("swapfile_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager)); |
| 337 | |
| 338 | /* |
| 339 | * Gather in a UPL all the VM pages requested by VM. |
| 340 | */ |
| 341 | mo_control = pager->swp_pgr_hdr.mo_control; |
| 342 | |
| 343 | upl_size = length; |
| 344 | upl_flags = |
| 345 | UPL_RET_ONLY_ABSENT | |
| 346 | UPL_SET_LITE | |
| 347 | UPL_NO_SYNC | |
| 348 | UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ |
| 349 | UPL_SET_INTERNAL; |
| 350 | pl_count = 0; |
| 351 | kr = memory_object_upl_request(mo_control, |
| 352 | offset, upl_size, |
| 353 | &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_OSFMK); |
| 354 | if (kr != KERN_SUCCESS) { |
| 355 | retval = kr; |
| 356 | goto done; |
| 357 | } |
| 358 | dst_object = mo_control->moc_object; |
| 359 | assert(dst_object != VM_OBJECT_NULL); |
| 360 | |
| 361 | |
| 362 | /* |
| 363 | * Reserve a virtual page in the kernel address space to map each |
| 364 | * destination physical page when it's its turn to be processed. |
| 365 | */ |
| 366 | vm_object_reference(kernel_object); /* ref. for mapping */ |
| 367 | kr = vm_map_find_space(kernel_map, |
| 368 | &kernel_mapping, |
| 369 | PAGE_SIZE_64, |
| 370 | 0, |
| 371 | 0, |
| 372 | VM_MAP_KERNEL_FLAGS_NONE, |
| 373 | VM_KERN_MEMORY_NONE, |
| 374 | &map_entry); |
| 375 | if (kr != KERN_SUCCESS) { |
| 376 | vm_object_deallocate(kernel_object); |
| 377 | retval = kr; |
| 378 | goto done; |
| 379 | } |
| 380 | VME_OBJECT_SET(map_entry, kernel_object); |
| 381 | VME_OFFSET_SET(map_entry, kernel_mapping - VM_MIN_KERNEL_ADDRESS); |
| 382 | vm_map_unlock(kernel_map); |
| 383 | dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping); |
| 384 | dst_ptr = (char *) dst_vaddr; |
| 385 | |
| 386 | /* |
| 387 | * Fill in the contents of the pages requested by VM. |
| 388 | */ |
| 389 | upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl); |
| 390 | pl_count = length / PAGE_SIZE; |
| 391 | for (cur_offset = 0; cur_offset < length; cur_offset += PAGE_SIZE) { |
| 392 | ppnum_t dst_pnum; |
| 393 | |
| 394 | if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) { |
| 395 | /* this page is not in the UPL: skip it */ |
| 396 | continue; |
| 397 | } |
| 398 | |
| 399 | /* |
| 400 | * Establish an explicit pmap mapping of the destination |
| 401 | * physical page. |
| 402 | * We can't do a regular VM mapping because the VM page |
| 403 | * is "busy". |
| 404 | */ |
| 405 | dst_pnum = (ppnum_t) |
| 406 | upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); |
| 407 | assert(dst_pnum != 0); |
| 408 | retval = pmap_enter(kernel_pmap, |
| 409 | kernel_mapping, |
| 410 | dst_pnum, |
| 411 | VM_PROT_READ | VM_PROT_WRITE, |
| 412 | VM_PROT_NONE, |
| 413 | 0, |
| 414 | TRUE); |
| 415 | |
| 416 | assert(retval == KERN_SUCCESS); |
| 417 | |
| 418 | if (retval != KERN_SUCCESS) { |
| 419 | goto done; |
| 420 | } |
| 421 | |
| 422 | memset(dst_ptr, '\0', PAGE_SIZE); |
| 423 | /* add an end-of-line to keep line counters happy */ |
| 424 | dst_ptr[PAGE_SIZE-1] = '\n'; |
| 425 | |
| 426 | /* |
| 427 | * Remove the pmap mapping of the destination page |
| 428 | * in the kernel. |
| 429 | */ |
| 430 | pmap_remove(kernel_pmap, |
| 431 | (addr64_t) kernel_mapping, |
| 432 | (addr64_t) (kernel_mapping + PAGE_SIZE_64)); |
| 433 | |
| 434 | } |
| 435 | |
| 436 | retval = KERN_SUCCESS; |
| 437 | done: |
| 438 | if (upl != NULL) { |
| 439 | /* clean up the UPL */ |
| 440 | |
| 441 | /* |
| 442 | * The pages are currently dirty because we've just been |
| 443 | * writing on them, but as far as we're concerned, they're |
| 444 | * clean since they contain their "original" contents as |
| 445 | * provided by us, the pager. |
| 446 | * Tell the UPL to mark them "clean". |
| 447 | */ |
| 448 | upl_clear_dirty(upl, TRUE); |
| 449 | |
| 450 | /* abort or commit the UPL */ |
| 451 | if (retval != KERN_SUCCESS) { |
| 452 | upl_abort(upl, 0); |
| 453 | } else { |
| 454 | boolean_t empty; |
| 455 | upl_commit_range(upl, 0, upl->size, |
| 456 | UPL_COMMIT_CS_VALIDATED, |
| 457 | upl_pl, pl_count, &empty); |
| 458 | } |
| 459 | |
| 460 | /* and deallocate the UPL */ |
| 461 | upl_deallocate(upl); |
| 462 | upl = NULL; |
| 463 | } |
| 464 | if (kernel_mapping != 0) { |
| 465 | /* clean up the mapping of the source and destination pages */ |
| 466 | kr = vm_map_remove(kernel_map, |
| 467 | kernel_mapping, |
| 468 | kernel_mapping + PAGE_SIZE_64, |
| 469 | VM_MAP_NO_FLAGS); |
| 470 | assert(kr == KERN_SUCCESS); |
| 471 | kernel_mapping = 0; |
| 472 | dst_vaddr = 0; |
| 473 | } |
| 474 | |
| 475 | return retval; |
| 476 | } |
| 477 | |
| 478 | /* |
| 479 | * swapfile_pager_reference() |
| 480 | * |
| 481 | * Get a reference on this memory object. |
| 482 | * For external usage only. Assumes that the initial reference count is not 0, |
| 483 | * i.e one should not "revive" a dead pager this way. |
| 484 | */ |
| 485 | void |
| 486 | swapfile_pager_reference( |
| 487 | memory_object_t mem_obj) |
| 488 | { |
| 489 | swapfile_pager_t pager; |
| 490 | |
| 491 | pager = swapfile_pager_lookup(mem_obj); |
| 492 | |
| 493 | lck_mtx_lock(&swapfile_pager_lock); |
| 494 | assert(pager->ref_count > 0); |
| 495 | pager->ref_count++; |
| 496 | lck_mtx_unlock(&swapfile_pager_lock); |
| 497 | } |
| 498 | |
| 499 | |
| 500 | /* |
| 501 | * swapfile_pager_dequeue: |
| 502 | * |
| 503 | * Removes a pager from the list of pagers. |
| 504 | * |
| 505 | * The caller must hold "swapfile_pager_lock". |
| 506 | */ |
| 507 | void |
| 508 | swapfile_pager_dequeue( |
| 509 | swapfile_pager_t pager) |
| 510 | { |
| 511 | assert(!pager->is_mapped); |
| 512 | |
| 513 | queue_remove(&swapfile_pager_queue, |
| 514 | pager, |
| 515 | swapfile_pager_t, |
| 516 | pager_queue); |
| 517 | pager->pager_queue.next = NULL; |
| 518 | pager->pager_queue.prev = NULL; |
| 519 | |
| 520 | swapfile_pager_count--; |
| 521 | } |
| 522 | |
| 523 | /* |
| 524 | * swapfile_pager_terminate_internal: |
| 525 | * |
| 526 | * Trigger the asynchronous termination of the memory object associated |
| 527 | * with this pager. |
| 528 | * When the memory object is terminated, there will be one more call |
| 529 | * to memory_object_deallocate() (i.e. swapfile_pager_deallocate()) |
| 530 | * to finish the clean up. |
| 531 | * |
| 532 | * "swapfile_pager_lock" should not be held by the caller. |
| 533 | * We don't need the lock because the pager has already been removed from |
| 534 | * the pagers' list and is now ours exclusively. |
| 535 | */ |
| 536 | void |
| 537 | swapfile_pager_terminate_internal( |
| 538 | swapfile_pager_t pager) |
| 539 | { |
| 540 | assert(pager->is_ready); |
| 541 | assert(!pager->is_mapped); |
| 542 | |
| 543 | if (pager->swapfile_vnode != NULL) { |
| 544 | pager->swapfile_vnode = NULL; |
| 545 | } |
| 546 | |
| 547 | /* trigger the destruction of the memory object */ |
| 548 | memory_object_destroy(pager->swp_pgr_hdr.mo_control, 0); |
| 549 | } |
| 550 | |
| 551 | /* |
| 552 | * swapfile_pager_deallocate_internal() |
| 553 | * |
| 554 | * Release a reference on this pager and free it when the last |
| 555 | * reference goes away. |
| 556 | * Can be called with swapfile_pager_lock held or not but always returns |
| 557 | * with it unlocked. |
| 558 | */ |
| 559 | void |
| 560 | swapfile_pager_deallocate_internal( |
| 561 | swapfile_pager_t pager, |
| 562 | boolean_t locked) |
| 563 | { |
| 564 | if (! locked) { |
| 565 | lck_mtx_lock(&swapfile_pager_lock); |
| 566 | } |
| 567 | |
| 568 | /* drop a reference on this pager */ |
| 569 | pager->ref_count--; |
| 570 | |
| 571 | if (pager->ref_count == 1) { |
| 572 | /* |
| 573 | * Only the "named" reference is left, which means that |
| 574 | * no one is really holding on to this pager anymore. |
| 575 | * Terminate it. |
| 576 | */ |
| 577 | swapfile_pager_dequeue(pager); |
| 578 | /* the pager is all ours: no need for the lock now */ |
| 579 | lck_mtx_unlock(&swapfile_pager_lock); |
| 580 | swapfile_pager_terminate_internal(pager); |
| 581 | } else if (pager->ref_count == 0) { |
| 582 | /* |
| 583 | * Dropped the existence reference; the memory object has |
| 584 | * been terminated. Do some final cleanup and release the |
| 585 | * pager structure. |
| 586 | */ |
| 587 | lck_mtx_unlock(&swapfile_pager_lock); |
| 588 | if (pager->swp_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { |
| 589 | memory_object_control_deallocate(pager->swp_pgr_hdr.mo_control); |
| 590 | pager->swp_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; |
| 591 | } |
| 592 | kfree(pager, sizeof (*pager)); |
| 593 | pager = SWAPFILE_PAGER_NULL; |
| 594 | } else { |
| 595 | /* there are still plenty of references: keep going... */ |
| 596 | lck_mtx_unlock(&swapfile_pager_lock); |
| 597 | } |
| 598 | |
| 599 | /* caution: lock is not held on return... */ |
| 600 | } |
| 601 | |
| 602 | /* |
| 603 | * swapfile_pager_deallocate() |
| 604 | * |
| 605 | * Release a reference on this pager and free it when the last |
| 606 | * reference goes away. |
| 607 | */ |
| 608 | void |
| 609 | swapfile_pager_deallocate( |
| 610 | memory_object_t mem_obj) |
| 611 | { |
| 612 | swapfile_pager_t pager; |
| 613 | |
| 614 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_deallocate: %p\n", mem_obj)); |
| 615 | pager = swapfile_pager_lookup(mem_obj); |
| 616 | swapfile_pager_deallocate_internal(pager, FALSE); |
| 617 | } |
| 618 | |
| 619 | /* |
| 620 | * |
| 621 | */ |
| 622 | kern_return_t |
| 623 | swapfile_pager_terminate( |
| 624 | #if !DEBUG |
| 625 | __unused |
| 626 | #endif |
| 627 | memory_object_t mem_obj) |
| 628 | { |
| 629 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_terminate: %p\n", mem_obj)); |
| 630 | |
| 631 | return KERN_SUCCESS; |
| 632 | } |
| 633 | |
| 634 | /* |
| 635 | * |
| 636 | */ |
| 637 | kern_return_t |
| 638 | swapfile_pager_synchronize( |
| 639 | __unused memory_object_t mem_obbj, |
| 640 | __unused memory_object_offset_t offset, |
| 641 | __unused memory_object_size_t length, |
| 642 | __unused vm_sync_t sync_flags) |
| 643 | { |
| 644 | panic("swapfile_pager_synchronize: memory_object_synchronize no longer supported\n"); |
| 645 | return (KERN_FAILURE); |
| 646 | } |
| 647 | |
| 648 | /* |
| 649 | * swapfile_pager_map() |
| 650 | * |
| 651 | * This allows VM to let us, the EMM, know that this memory object |
| 652 | * is currently mapped one or more times. This is called by VM each time |
| 653 | * the memory object gets mapped and we take one extra reference on the |
| 654 | * memory object to account for all its mappings. |
| 655 | */ |
| 656 | kern_return_t |
| 657 | swapfile_pager_map( |
| 658 | memory_object_t mem_obj, |
| 659 | __unused vm_prot_t prot) |
| 660 | { |
| 661 | swapfile_pager_t pager; |
| 662 | |
| 663 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_map: %p\n", mem_obj)); |
| 664 | |
| 665 | pager = swapfile_pager_lookup(mem_obj); |
| 666 | |
| 667 | lck_mtx_lock(&swapfile_pager_lock); |
| 668 | assert(pager->is_ready); |
| 669 | assert(pager->ref_count > 0); /* pager is alive */ |
| 670 | if (pager->is_mapped == FALSE) { |
| 671 | /* |
| 672 | * First mapping of this pager: take an extra reference |
| 673 | * that will remain until all the mappings of this pager |
| 674 | * are removed. |
| 675 | */ |
| 676 | pager->is_mapped = TRUE; |
| 677 | pager->ref_count++; |
| 678 | } |
| 679 | lck_mtx_unlock(&swapfile_pager_lock); |
| 680 | |
| 681 | return KERN_SUCCESS; |
| 682 | } |
| 683 | |
| 684 | /* |
| 685 | * swapfile_pager_last_unmap() |
| 686 | * |
| 687 | * This is called by VM when this memory object is no longer mapped anywhere. |
| 688 | */ |
| 689 | kern_return_t |
| 690 | swapfile_pager_last_unmap( |
| 691 | memory_object_t mem_obj) |
| 692 | { |
| 693 | swapfile_pager_t pager; |
| 694 | |
| 695 | PAGER_DEBUG(PAGER_ALL, |
| 696 | ("swapfile_pager_last_unmap: %p\n", mem_obj)); |
| 697 | |
| 698 | pager = swapfile_pager_lookup(mem_obj); |
| 699 | |
| 700 | lck_mtx_lock(&swapfile_pager_lock); |
| 701 | if (pager->is_mapped) { |
| 702 | /* |
| 703 | * All the mappings are gone, so let go of the one extra |
| 704 | * reference that represents all the mappings of this pager. |
| 705 | */ |
| 706 | pager->is_mapped = FALSE; |
| 707 | swapfile_pager_deallocate_internal(pager, TRUE); |
| 708 | /* caution: deallocate_internal() released the lock ! */ |
| 709 | } else { |
| 710 | lck_mtx_unlock(&swapfile_pager_lock); |
| 711 | } |
| 712 | |
| 713 | return KERN_SUCCESS; |
| 714 | } |
| 715 | |
| 716 | |
| 717 | /* |
| 718 | * |
| 719 | */ |
| 720 | swapfile_pager_t |
| 721 | swapfile_pager_lookup( |
| 722 | memory_object_t mem_obj) |
| 723 | { |
| 724 | swapfile_pager_t pager; |
| 725 | |
| 726 | assert(mem_obj->mo_pager_ops == &swapfile_pager_ops); |
| 727 | __IGNORE_WCASTALIGN(pager = (swapfile_pager_t) mem_obj); |
| 728 | assert(pager->ref_count > 0); |
| 729 | return pager; |
| 730 | } |
| 731 | |
| 732 | swapfile_pager_t |
| 733 | swapfile_pager_create( |
| 734 | struct vnode *vp) |
| 735 | { |
| 736 | swapfile_pager_t pager, pager2; |
| 737 | memory_object_control_t control; |
| 738 | kern_return_t kr; |
| 739 | |
| 740 | pager = (swapfile_pager_t) kalloc(sizeof (*pager)); |
| 741 | if (pager == SWAPFILE_PAGER_NULL) { |
| 742 | return SWAPFILE_PAGER_NULL; |
| 743 | } |
| 744 | |
| 745 | /* |
| 746 | * The vm_map call takes both named entry ports and raw memory |
| 747 | * objects in the same parameter. We need to make sure that |
| 748 | * vm_map does not see this object as a named entry port. So, |
| 749 | * we reserve the second word in the object for a fake ip_kotype |
| 750 | * setting - that will tell vm_map to use it as a memory object. |
| 751 | */ |
| 752 | pager->swp_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; |
| 753 | pager->swp_pgr_hdr.mo_pager_ops = &swapfile_pager_ops; |
| 754 | pager->swp_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; |
| 755 | |
| 756 | pager->is_ready = FALSE;/* not ready until it has a "name" */ |
| 757 | pager->ref_count = 1; /* setup reference */ |
| 758 | pager->is_mapped = FALSE; |
| 759 | pager->swapfile_vnode = vp; |
| 760 | |
| 761 | lck_mtx_lock(&swapfile_pager_lock); |
| 762 | /* see if anyone raced us to create a pager for the same object */ |
| 763 | queue_iterate(&swapfile_pager_queue, |
| 764 | pager2, |
| 765 | swapfile_pager_t, |
| 766 | pager_queue) { |
| 767 | if (pager2->swapfile_vnode == vp) { |
| 768 | break; |
| 769 | } |
| 770 | } |
| 771 | if (! queue_end(&swapfile_pager_queue, |
| 772 | (queue_entry_t) pager2)) { |
| 773 | /* while we hold the lock, transfer our setup ref to winner */ |
| 774 | pager2->ref_count++; |
| 775 | /* we lost the race, down with the loser... */ |
| 776 | lck_mtx_unlock(&swapfile_pager_lock); |
| 777 | pager->swapfile_vnode = NULL; |
| 778 | kfree(pager, sizeof (*pager)); |
| 779 | /* ... and go with the winner */ |
| 780 | pager = pager2; |
| 781 | /* let the winner make sure the pager gets ready */ |
| 782 | return pager; |
| 783 | } |
| 784 | |
| 785 | /* enter new pager at the head of our list of pagers */ |
| 786 | queue_enter_first(&swapfile_pager_queue, |
| 787 | pager, |
| 788 | swapfile_pager_t, |
| 789 | pager_queue); |
| 790 | swapfile_pager_count++; |
| 791 | if (swapfile_pager_count > swapfile_pager_count_max) { |
| 792 | swapfile_pager_count_max = swapfile_pager_count; |
| 793 | } |
| 794 | lck_mtx_unlock(&swapfile_pager_lock); |
| 795 | |
| 796 | kr = memory_object_create_named((memory_object_t) pager, |
| 797 | 0, |
| 798 | &control); |
| 799 | assert(kr == KERN_SUCCESS); |
| 800 | |
| 801 | lck_mtx_lock(&swapfile_pager_lock); |
| 802 | /* the new pager is now ready to be used */ |
| 803 | pager->is_ready = TRUE; |
| 804 | lck_mtx_unlock(&swapfile_pager_lock); |
| 805 | |
| 806 | /* wakeup anyone waiting for this pager to be ready */ |
| 807 | thread_wakeup(&pager->is_ready); |
| 808 | |
| 809 | return pager; |
| 810 | } |
| 811 | |
| 812 | /* |
| 813 | * swapfile_pager_setup() |
| 814 | * |
| 815 | * Provide the caller with a memory object backed by the provided |
| 816 | * "backing_object" VM object. If such a memory object already exists, |
| 817 | * re-use it, otherwise create a new memory object. |
| 818 | */ |
| 819 | memory_object_t |
| 820 | swapfile_pager_setup( |
| 821 | struct vnode *vp) |
| 822 | { |
| 823 | swapfile_pager_t pager; |
| 824 | |
| 825 | lck_mtx_lock(&swapfile_pager_lock); |
| 826 | |
| 827 | queue_iterate(&swapfile_pager_queue, |
| 828 | pager, |
| 829 | swapfile_pager_t, |
| 830 | pager_queue) { |
| 831 | if (pager->swapfile_vnode == vp) { |
| 832 | break; |
| 833 | } |
| 834 | } |
| 835 | if (queue_end(&swapfile_pager_queue, |
| 836 | (queue_entry_t) pager)) { |
| 837 | /* no existing pager for this backing object */ |
| 838 | pager = SWAPFILE_PAGER_NULL; |
| 839 | } else { |
| 840 | /* make sure pager doesn't disappear */ |
| 841 | pager->ref_count++; |
| 842 | } |
| 843 | |
| 844 | lck_mtx_unlock(&swapfile_pager_lock); |
| 845 | |
| 846 | if (pager == SWAPFILE_PAGER_NULL) { |
| 847 | pager = swapfile_pager_create(vp); |
| 848 | if (pager == SWAPFILE_PAGER_NULL) { |
| 849 | return MEMORY_OBJECT_NULL; |
| 850 | } |
| 851 | } |
| 852 | |
| 853 | lck_mtx_lock(&swapfile_pager_lock); |
| 854 | while (!pager->is_ready) { |
| 855 | lck_mtx_sleep(&swapfile_pager_lock, |
| 856 | LCK_SLEEP_DEFAULT, |
| 857 | &pager->is_ready, |
| 858 | THREAD_UNINT); |
| 859 | } |
| 860 | lck_mtx_unlock(&swapfile_pager_lock); |
| 861 | |
| 862 | return (memory_object_t) pager; |
| 863 | } |
| 864 | |
| 865 | memory_object_control_t |
| 866 | swapfile_pager_control( |
| 867 | memory_object_t mem_obj) |
| 868 | { |
| 869 | swapfile_pager_t pager; |
| 870 | |
| 871 | if (mem_obj == MEMORY_OBJECT_NULL || |
| 872 | mem_obj->mo_pager_ops != &swapfile_pager_ops) { |
| 873 | return MEMORY_OBJECT_CONTROL_NULL; |
| 874 | } |
| 875 | pager = swapfile_pager_lookup(mem_obj); |
| 876 | return pager->swp_pgr_hdr.mo_control; |
| 877 | } |