]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_apple_protect.c
projects / apple / xnu.git / blob
? search:


e5945fbad1b9027b440a5e343e28441d42078057
[apple/xnu.git] / osfmk / vm / vm_apple_protect.c
1 /*
2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/errno.h>
30
31 #include <mach/mach_types.h>
32 #include <mach/mach_traps.h>
33 #include <mach/host_priv.h>
34 #include <mach/kern_return.h>
35 #include <mach/memory_object_control.h>
36 #include <mach/memory_object_types.h>
37 #include <mach/port.h>
38 #include <mach/policy.h>
39 #include <mach/upl.h>
40 #include <mach/thread_act.h>
41 #include <mach/mach_vm.h>
42
43 #include <kern/host.h>
44 #include <kern/kalloc.h>
45 #include <kern/page_decrypt.h>
46 #include <kern/queue.h>
47 #include <kern/thread.h>
48
49 #include <ipc/ipc_port.h>
50 #include <ipc/ipc_space.h>
51
52 #include <default_pager/default_pager_types.h>
53 #include <default_pager/default_pager_object_server.h>
54
55 #include <vm/vm_fault.h>
56 #include <vm/vm_map.h>
57 #include <vm/vm_pageout.h>
58 #include <vm/memory_object.h>
59 #include <vm/vm_pageout.h>
60 #include <vm/vm_protos.h>
61
62
63 /*
64 * APPLE PROTECT MEMORY PAGER
65 *
66 * This external memory manager (EMM) handles memory from the encrypted
67 * sections of some executables protected by the DSMOS kernel extension.
68 *
69 * It mostly handles page-in requests (from memory_object_data_request()) by
70 * getting the encrypted data from its backing VM object, itself backed by
71 * the encrypted file, decrypting it and providing it to VM.
72 *
73 * The decrypted pages will never be dirtied, so the memory manager doesn't
74 * need to handle page-out requests (from memory_object_data_return()). The
75 * pages need to be mapped copy-on-write, so that the originals stay clean.
76 *
77 * We don't expect to have to handle a large number of apple-protected
78 * binaries, so the data structures are very simple (simple linked list)
79 * for now.
80 */
81
82 /* forward declarations */
83 void apple_protect_pager_reference(memory_object_t mem_obj);
84 void apple_protect_pager_deallocate(memory_object_t mem_obj);
85 kern_return_t apple_protect_pager_init(memory_object_t mem_obj,
86 memory_object_control_t control,
87 memory_object_cluster_size_t pg_size);
88 kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj);
89 kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj,
90 memory_object_offset_t offset,
91 memory_object_cluster_size_t length,
92 vm_prot_t protection_required,
93 memory_object_fault_info_t fault_info);
94 kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj,
95 memory_object_offset_t offset,
96 memory_object_cluster_size_t data_cnt,
97 memory_object_offset_t *resid_offset,
98 int *io_error,
99 boolean_t dirty,
100 boolean_t kernel_copy,
101 int upl_flags);
102 kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj,
103 memory_object_offset_t offset,
104 memory_object_cluster_size_t data_cnt);
105 kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj,
106 memory_object_offset_t offset,
107 memory_object_size_t size,
108 vm_prot_t desired_access);
109 kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj,
110 memory_object_offset_t offset,
111 memory_object_size_t length,
112 vm_sync_t sync_flags);
113 kern_return_t apple_protect_pager_map(memory_object_t mem_obj,
114 vm_prot_t prot);
115 kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj);
116
117 /*
118 * Vector of VM operations for this EMM.
119 * These routines are invoked by VM via the memory_object_*() interfaces.
120 */
121 const struct memory_object_pager_ops apple_protect_pager_ops = {
122 apple_protect_pager_reference,
123 apple_protect_pager_deallocate,
124 apple_protect_pager_init,
125 apple_protect_pager_terminate,
126 apple_protect_pager_data_request,
127 apple_protect_pager_data_return,
128 apple_protect_pager_data_initialize,
129 apple_protect_pager_data_unlock,
130 apple_protect_pager_synchronize,
131 apple_protect_pager_map,
132 apple_protect_pager_last_unmap,
133 NULL, /* data_reclaim */
134 "apple protect pager"
135 };
136
137 /*
138 * The "apple_protect_pager" describes a memory object backed by
139 * the "apple protect" EMM.
140 */
141 typedef struct apple_protect_pager {
142 struct ipc_object_header pager_header; /* fake ip_kotype() */
143 memory_object_pager_ops_t pager_ops; /* == &apple_protect_pager_ops */
144 queue_chain_t pager_queue; /* next & prev pagers */
145 unsigned int ref_count; /* reference count */
146 boolean_t is_ready; /* is this pager ready ? */
147 boolean_t is_mapped; /* is this mem_obj mapped ? */
148 memory_object_control_t pager_control; /* mem object control handle */
149 vm_object_t backing_object; /* VM obj w/ encrypted data */
150 struct pager_crypt_info crypt;
151 } *apple_protect_pager_t;
152 #define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL)
153 #define pager_ikot pager_header.io_bits
154
155 /*
156 * List of memory objects managed by this EMM.
157 * The list is protected by the "apple_protect_pager_lock" lock.
158 */
159 int apple_protect_pager_count = 0; /* number of pagers */
160 int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */
161 queue_head_t apple_protect_pager_queue;
162 decl_lck_mtx_data(,apple_protect_pager_lock)
163
164 /*
165 * Maximum number of unmapped pagers we're willing to keep around.
166 */
167 int apple_protect_pager_cache_limit = 10;
168
169 /*
170 * Statistics & counters.
171 */
172 int apple_protect_pager_count_max = 0;
173 int apple_protect_pager_count_unmapped_max = 0;
174 int apple_protect_pager_num_trim_max = 0;
175 int apple_protect_pager_num_trim_total = 0;
176
177
178 lck_grp_t apple_protect_pager_lck_grp;
179 lck_grp_attr_t apple_protect_pager_lck_grp_attr;
180 lck_attr_t apple_protect_pager_lck_attr;
181
182
183 /* internal prototypes */
184 apple_protect_pager_t apple_protect_pager_create(vm_object_t backing_object,
185 struct pager_crypt_info *crypt_info);
186 apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj);
187 void apple_protect_pager_dequeue(apple_protect_pager_t pager);
188 void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager,
189 boolean_t locked);
190 void apple_protect_pager_terminate_internal(apple_protect_pager_t pager);
191 void apple_protect_pager_trim(void);
192
193
194 #if DEBUG
195 int apple_protect_pagerdebug = 0;
196 #define PAGER_ALL 0xffffffff
197 #define PAGER_INIT 0x00000001
198 #define PAGER_PAGEIN 0x00000002
199
200 #define PAGER_DEBUG(LEVEL, A) \
201 MACRO_BEGIN \
202 if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \
203 printf A; \
204 } \
205 MACRO_END
206 #else
207 #define PAGER_DEBUG(LEVEL, A)
208 #endif
209
210
211 void
212 apple_protect_pager_bootstrap(void)
213 {
214 lck_grp_attr_setdefault(&apple_protect_pager_lck_grp_attr);
215 lck_grp_init(&apple_protect_pager_lck_grp, "apple_protect", &apple_protect_pager_lck_grp_attr);
216 lck_attr_setdefault(&apple_protect_pager_lck_attr);
217 lck_mtx_init(&apple_protect_pager_lock, &apple_protect_pager_lck_grp, &apple_protect_pager_lck_attr);
218 queue_init(&apple_protect_pager_queue);
219 }
220
221 /*
222 * apple_protect_pager_init()
223 *
224 * Initialize the memory object and makes it ready to be used and mapped.
225 */
226 kern_return_t
227 apple_protect_pager_init(
228 memory_object_t mem_obj,
229 memory_object_control_t control,
230 #if !DEBUG
231 __unused
232 #endif
233 memory_object_cluster_size_t pg_size)
234 {
235 apple_protect_pager_t pager;
236 kern_return_t kr;
237 memory_object_attr_info_data_t attributes;
238
239 PAGER_DEBUG(PAGER_ALL,
240 ("apple_protect_pager_init: %p, %p, %x\n",
241 mem_obj, control, pg_size));
242
243 if (control == MEMORY_OBJECT_CONTROL_NULL)
244 return KERN_INVALID_ARGUMENT;
245
246 pager = apple_protect_pager_lookup(mem_obj);
247
248 memory_object_control_reference(control);
249
250 pager->pager_control = control;
251
252 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
253 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
254 attributes.cluster_size = (1 << (PAGE_SHIFT));
255 attributes.may_cache_object = FALSE;
256 attributes.temporary = TRUE;
257
258 kr = memory_object_change_attributes(
259 control,
260 MEMORY_OBJECT_ATTRIBUTE_INFO,
261 (memory_object_info_t) &attributes,
262 MEMORY_OBJECT_ATTR_INFO_COUNT);
263 if (kr != KERN_SUCCESS)
264 panic("apple_protect_pager_init: "
265 "memory_object_change_attributes() failed");
266
267 return KERN_SUCCESS;
268 }
269
270 /*
271 * apple_protect_data_return()
272 *
273 * Handles page-out requests from VM. This should never happen since
274 * the pages provided by this EMM are not supposed to be dirty or dirtied
275 * and VM should simply discard the contents and reclaim the pages if it
276 * needs to.
277 */
278 kern_return_t
279 apple_protect_pager_data_return(
280 __unused memory_object_t mem_obj,
281 __unused memory_object_offset_t offset,
282 __unused memory_object_cluster_size_t data_cnt,
283 __unused memory_object_offset_t *resid_offset,
284 __unused int *io_error,
285 __unused boolean_t dirty,
286 __unused boolean_t kernel_copy,
287 __unused int upl_flags)
288 {
289 panic("apple_protect_pager_data_return: should never get called");
290 return KERN_FAILURE;
291 }
292
293 kern_return_t
294 apple_protect_pager_data_initialize(
295 __unused memory_object_t mem_obj,
296 __unused memory_object_offset_t offset,
297 __unused memory_object_cluster_size_t data_cnt)
298 {
299 panic("apple_protect_pager_data_initialize: should never get called");
300 return KERN_FAILURE;
301 }
302
303 kern_return_t
304 apple_protect_pager_data_unlock(
305 __unused memory_object_t mem_obj,
306 __unused memory_object_offset_t offset,
307 __unused memory_object_size_t size,
308 __unused vm_prot_t desired_access)
309 {
310 return KERN_FAILURE;
311 }
312
313 /*
314 * apple_protect_pager_data_request()
315 *
316 * Handles page-in requests from VM.
317 */
318 kern_return_t
319 apple_protect_pager_data_request(
320 memory_object_t mem_obj,
321 memory_object_offset_t offset,
322 memory_object_cluster_size_t length,
323 #if !DEBUG
324 __unused
325 #endif
326 vm_prot_t protection_required,
327 memory_object_fault_info_t mo_fault_info)
328 {
329 apple_protect_pager_t pager;
330 memory_object_control_t mo_control;
331 upl_t upl;
332 int upl_flags;
333 upl_size_t upl_size;
334 upl_page_info_t *upl_pl;
335 unsigned int pl_count;
336 vm_object_t src_object, dst_object;
337 kern_return_t kr, retval;
338 vm_map_offset_t kernel_mapping;
339 vm_offset_t src_vaddr, dst_vaddr;
340 vm_offset_t cur_offset;
341 vm_offset_t offset_in_page;
342 vm_map_entry_t map_entry;
343 kern_return_t error_code;
344 vm_prot_t prot;
345 vm_page_t src_page, top_page;
346 int interruptible;
347 struct vm_object_fault_info fault_info;
348 int ret;
349
350 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required));
351
352 retval = KERN_SUCCESS;
353 src_object = VM_OBJECT_NULL;
354 kernel_mapping = 0;
355 upl = NULL;
356 upl_pl = NULL;
357 fault_info = *((struct vm_object_fault_info *) mo_fault_info);
358 fault_info.stealth = TRUE;
359 fault_info.io_sync = FALSE;
360 fault_info.mark_zf_absent = FALSE;
361 fault_info.batch_pmap_op = FALSE;
362 interruptible = fault_info.interruptible;
363
364 pager = apple_protect_pager_lookup(mem_obj);
365 assert(pager->is_ready);
366 assert(pager->ref_count > 1); /* pager is alive and mapped */
367
368 PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
369
370 /*
371 * Gather in a UPL all the VM pages requested by VM.
372 */
373 mo_control = pager->pager_control;
374
375 upl_size = length;
376 upl_flags =
377 UPL_RET_ONLY_ABSENT |
378 UPL_SET_LITE |
379 UPL_NO_SYNC |
380 UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */
381 UPL_SET_INTERNAL;
382 pl_count = 0;
383 kr = memory_object_upl_request(mo_control,
384 offset, upl_size,
385 &upl, NULL, NULL, upl_flags);
386 if (kr != KERN_SUCCESS) {
387 retval = kr;
388 goto done;
389 }
390 dst_object = mo_control->moc_object;
391 assert(dst_object != VM_OBJECT_NULL);
392
393
394 /*
395 * Reserve 2 virtual pages in the kernel address space to map each
396 * source and destination physical pages when it's their turn to
397 * be processed.
398 */
399 vm_object_reference(kernel_object); /* ref. for mapping */
400 kr = vm_map_find_space(kernel_map,
401 &kernel_mapping,
402 2 * PAGE_SIZE_64,
403 0,
404 0,
405 &map_entry);
406 if (kr != KERN_SUCCESS) {
407 vm_object_deallocate(kernel_object);
408 retval = kr;
409 goto done;
410 }
411 map_entry->object.vm_object = kernel_object;
412 map_entry->offset = kernel_mapping;
413 vm_map_unlock(kernel_map);
414 src_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping);
415 dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping + PAGE_SIZE_64);
416
417 /*
418 * We'll map the encrypted data in the kernel address space from the
419 * backing VM object (itself backed by the encrypted file via
420 * the vnode pager).
421 */
422 src_object = pager->backing_object;
423 assert(src_object != VM_OBJECT_NULL);
424 vm_object_reference(src_object); /* to keep the source object alive */
425
426 /*
427 * Fill in the contents of the pages requested by VM.
428 */
429 upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
430 pl_count = length / PAGE_SIZE;
431 for (cur_offset = 0;
432 retval == KERN_SUCCESS && cur_offset < length;
433 cur_offset += PAGE_SIZE) {
434 ppnum_t dst_pnum;
435
436 if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) {
437 /* this page is not in the UPL: skip it */
438 continue;
439 }
440
441 /*
442 * Map the source (encrypted) page in the kernel's
443 * virtual address space.
444 * We already hold a reference on the src_object.
445 */
446 retry_src_fault:
447 vm_object_lock(src_object);
448 vm_object_paging_begin(src_object);
449 error_code = 0;
450 prot = VM_PROT_READ;
451 src_page = VM_PAGE_NULL;
452 kr = vm_fault_page(src_object,
453 offset + cur_offset,
454 VM_PROT_READ,
455 FALSE,
456 FALSE, /* src_page not looked up */
457 &prot,
458 &src_page,
459 &top_page,
460 NULL,
461 &error_code,
462 FALSE,
463 FALSE,
464 &fault_info);
465 switch (kr) {
466 case VM_FAULT_SUCCESS:
467 break;
468 case VM_FAULT_RETRY:
469 goto retry_src_fault;
470 case VM_FAULT_MEMORY_SHORTAGE:
471 if (vm_page_wait(interruptible)) {
472 goto retry_src_fault;
473 }
474 /* fall thru */
475 case VM_FAULT_INTERRUPTED:
476 retval = MACH_SEND_INTERRUPTED;
477 goto done;
478 case VM_FAULT_SUCCESS_NO_VM_PAGE:
479 /* success but no VM page: fail */
480 vm_object_paging_end(src_object);
481 vm_object_unlock(src_object);
482 /*FALLTHROUGH*/
483 case VM_FAULT_MEMORY_ERROR:
484 /* the page is not there ! */
485 if (error_code) {
486 retval = error_code;
487 } else {
488 retval = KERN_MEMORY_ERROR;
489 }
490 goto done;
491 default:
492 panic("apple_protect_pager_data_request: "
493 "vm_fault_page() unexpected error 0x%x\n",
494 kr);
495 }
496 assert(src_page != VM_PAGE_NULL);
497 assert(src_page->busy);
498
499 if (!src_page->active &&
500 !src_page->inactive &&
501 !src_page->throttled) {
502 vm_page_lockspin_queues();
503 if (!src_page->active &&
504 !src_page->inactive &&
505 !src_page->throttled) {
506 vm_page_deactivate(src_page);
507 }
508 vm_page_unlock_queues();
509 }
510
511 /*
512 * Establish an explicit mapping of the source
513 * physical page.
514 */
515 pmap_enter(kernel_pmap,
516 kernel_mapping,
517 src_page->phys_page,
518 VM_PROT_READ,
519 VM_PROT_NONE,
520 0,
521 TRUE);
522 /*
523 * Establish an explicit pmap mapping of the destination
524 * physical page.
525 * We can't do a regular VM mapping because the VM page
526 * is "busy".
527 */
528 dst_pnum = (ppnum_t)
529 upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));
530 assert(dst_pnum != 0);
531 pmap_enter(kernel_pmap,
532 kernel_mapping + PAGE_SIZE_64,
533 dst_pnum,
534 VM_PROT_READ | VM_PROT_WRITE,
535 VM_PROT_NONE,
536 0,
537 TRUE);
538
539 /*
540 * Decrypt the encrypted contents of the source page
541 * into the destination page.
542 */
543 for (offset_in_page = 0;
544 offset_in_page < PAGE_SIZE;
545 offset_in_page += 4096) {
546 ret = pager->crypt.page_decrypt((const void *)
547 (src_vaddr +
548 offset_in_page),
549 (void *)
550 (dst_vaddr +
551 offset_in_page),
552 (offset +
553 cur_offset +
554 offset_in_page),
555 pager->crypt.crypt_ops);
556 if (ret) {
557 break;
558 }
559 }
560 if (ret) {
561 /*
562 * Decryption failed. Abort the fault.
563 */
564 retval = KERN_ABORTED;
565 } else {
566 /*
567 * Validate the original page...
568 */
569 if (src_page->object->code_signed) {
570 vm_page_validate_cs_mapped(
571 src_page,
572 (const void *) src_vaddr);
573 }
574 /*
575 * ... and transfer the results to the destination page.
576 */
577 UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE,
578 src_page->cs_validated);
579 UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE,
580 src_page->cs_tainted);
581 UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE,
582 src_page->cs_nx);
583 }
584
585 /*
586 * Remove the pmap mapping of the source and destination pages
587 * in the kernel.
588 */
589 pmap_remove(kernel_pmap,
590 (addr64_t) kernel_mapping,
591 (addr64_t) (kernel_mapping + (2 * PAGE_SIZE_64)));
592
593 /*
594 * Cleanup the result of vm_fault_page() of the source page.
595 */
596 PAGE_WAKEUP_DONE(src_page);
597 vm_object_paging_end(src_page->object);
598 vm_object_unlock(src_page->object);
599 if (top_page != VM_PAGE_NULL) {
600 vm_object_t top_object;
601
602 top_object = top_page->object;
603 vm_object_lock(top_object);
604 VM_PAGE_FREE(top_page);
605 vm_object_paging_end(top_object);
606 vm_object_unlock(top_object);
607 }
608 }
609
610 done:
611 if (upl != NULL) {
612 /* clean up the UPL */
613
614 /*
615 * The pages are currently dirty because we've just been
616 * writing on them, but as far as we're concerned, they're
617 * clean since they contain their "original" contents as
618 * provided by us, the pager.
619 * Tell the UPL to mark them "clean".
620 */
621 upl_clear_dirty(upl, TRUE);
622
623 /* abort or commit the UPL */
624 if (retval != KERN_SUCCESS) {
625 upl_abort(upl, 0);
626 if (retval == KERN_ABORTED) {
627 wait_result_t wait_result;
628
629 /*
630 * We aborted the fault and did not provide
631 * any contents for the requested pages but
632 * the pages themselves are not invalid, so
633 * let's return success and let the caller
634 * retry the fault, in case it might succeed
635 * later (when the decryption code is up and
636 * running in the kernel, for example).
637 */
638 retval = KERN_SUCCESS;
639 /*
640 * Wait a little bit first to avoid using
641 * too much CPU time retrying and failing
642 * the same fault over and over again.
643 */
644 wait_result = assert_wait_timeout(
645 (event_t) apple_protect_pager_data_request,
646 THREAD_UNINT,
647 10000, /* 10ms */
648 NSEC_PER_USEC);
649 assert(wait_result == THREAD_WAITING);
650 wait_result = thread_block(THREAD_CONTINUE_NULL);
651 assert(wait_result == THREAD_TIMED_OUT);
652 }
653 } else {
654 boolean_t empty;
655 upl_commit_range(upl, 0, upl->size,
656 UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL,
657 upl_pl, pl_count, &empty);
658 }
659
660 /* and deallocate the UPL */
661 upl_deallocate(upl);
662 upl = NULL;
663 }
664 if (kernel_mapping != 0) {
665 /* clean up the mapping of the source and destination pages */
666 kr = vm_map_remove(kernel_map,
667 kernel_mapping,
668 kernel_mapping + (2 * PAGE_SIZE_64),
669 VM_MAP_NO_FLAGS);
670 assert(kr == KERN_SUCCESS);
671 kernel_mapping = 0;
672 src_vaddr = 0;
673 dst_vaddr = 0;
674 }
675 if (src_object != VM_OBJECT_NULL) {
676 vm_object_deallocate(src_object);
677 }
678
679 return retval;
680 }
681
682 /*
683 * apple_protect_pager_reference()
684 *
685 * Get a reference on this memory object.
686 * For external usage only. Assumes that the initial reference count is not 0,
687 * i.e one should not "revive" a dead pager this way.
688 */
689 void
690 apple_protect_pager_reference(
691 memory_object_t mem_obj)
692 {
693 apple_protect_pager_t pager;
694
695 pager = apple_protect_pager_lookup(mem_obj);
696
697 lck_mtx_lock(&apple_protect_pager_lock);
698 assert(pager->ref_count > 0);
699 pager->ref_count++;
700 lck_mtx_unlock(&apple_protect_pager_lock);
701 }
702
703
704 /*
705 * apple_protect_pager_dequeue:
706 *
707 * Removes a pager from the list of pagers.
708 *
709 * The caller must hold "apple_protect_pager_lock".
710 */
711 void
712 apple_protect_pager_dequeue(
713 apple_protect_pager_t pager)
714 {
715 assert(!pager->is_mapped);
716
717 queue_remove(&apple_protect_pager_queue,
718 pager,
719 apple_protect_pager_t,
720 pager_queue);
721 pager->pager_queue.next = NULL;
722 pager->pager_queue.prev = NULL;
723
724 apple_protect_pager_count--;
725 }
726
727 /*
728 * apple_protect_pager_terminate_internal:
729 *
730 * Trigger the asynchronous termination of the memory object associated
731 * with this pager.
732 * When the memory object is terminated, there will be one more call
733 * to memory_object_deallocate() (i.e. apple_protect_pager_deallocate())
734 * to finish the clean up.
735 *
736 * "apple_protect_pager_lock" should not be held by the caller.
737 * We don't need the lock because the pager has already been removed from
738 * the pagers' list and is now ours exclusively.
739 */
740 void
741 apple_protect_pager_terminate_internal(
742 apple_protect_pager_t pager)
743 {
744 assert(pager->is_ready);
745 assert(!pager->is_mapped);
746
747 if (pager->backing_object != VM_OBJECT_NULL) {
748 vm_object_deallocate(pager->backing_object);
749 pager->backing_object = VM_OBJECT_NULL;
750 }
751
752 /* deallocate any crypt module data */
753 if(pager->crypt.crypt_end)
754 pager->crypt.crypt_end(pager->crypt.crypt_ops);
755
756 /* trigger the destruction of the memory object */
757 memory_object_destroy(pager->pager_control, 0);
758 }
759
760 /*
761 * apple_protect_pager_deallocate_internal()
762 *
763 * Release a reference on this pager and free it when the last
764 * reference goes away.
765 * Can be called with apple_protect_pager_lock held or not but always returns
766 * with it unlocked.
767 */
768 void
769 apple_protect_pager_deallocate_internal(
770 apple_protect_pager_t pager,
771 boolean_t locked)
772 {
773 boolean_t needs_trimming;
774 int count_unmapped;
775
776 if (! locked) {
777 lck_mtx_lock(&apple_protect_pager_lock);
778 }
779
780 count_unmapped = (apple_protect_pager_count -
781 apple_protect_pager_count_mapped);
782 if (count_unmapped > apple_protect_pager_cache_limit) {
783 /* we have too many unmapped pagers: trim some */
784 needs_trimming = TRUE;
785 } else {
786 needs_trimming = FALSE;
787 }
788
789 /* drop a reference on this pager */
790 pager->ref_count--;
791
792 if (pager->ref_count == 1) {
793 /*
794 * Only the "named" reference is left, which means that
795 * no one is really holding on to this pager anymore.
796 * Terminate it.
797 */
798 apple_protect_pager_dequeue(pager);
799 /* the pager is all ours: no need for the lock now */
800 lck_mtx_unlock(&apple_protect_pager_lock);
801 apple_protect_pager_terminate_internal(pager);
802 } else if (pager->ref_count == 0) {
803 /*
804 * Dropped the existence reference; the memory object has
805 * been terminated. Do some final cleanup and release the
806 * pager structure.
807 */
808 lck_mtx_unlock(&apple_protect_pager_lock);
809 if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
810 memory_object_control_deallocate(pager->pager_control);
811 pager->pager_control = MEMORY_OBJECT_CONTROL_NULL;
812 }
813 kfree(pager, sizeof (*pager));
814 pager = APPLE_PROTECT_PAGER_NULL;
815 } else {
816 /* there are still plenty of references: keep going... */
817 lck_mtx_unlock(&apple_protect_pager_lock);
818 }
819
820 if (needs_trimming) {
821 apple_protect_pager_trim();
822 }
823 /* caution: lock is not held on return... */
824 }
825
826 /*
827 * apple_protect_pager_deallocate()
828 *
829 * Release a reference on this pager and free it when the last
830 * reference goes away.
831 */
832 void
833 apple_protect_pager_deallocate(
834 memory_object_t mem_obj)
835 {
836 apple_protect_pager_t pager;
837
838 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj));
839 pager = apple_protect_pager_lookup(mem_obj);
840 apple_protect_pager_deallocate_internal(pager, FALSE);
841 }
842
843 /*
844 *
845 */
846 kern_return_t
847 apple_protect_pager_terminate(
848 #if !DEBUG
849 __unused
850 #endif
851 memory_object_t mem_obj)
852 {
853 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj));
854
855 return KERN_SUCCESS;
856 }
857
858 /*
859 *
860 */
861 kern_return_t
862 apple_protect_pager_synchronize(
863 memory_object_t mem_obj,
864 memory_object_offset_t offset,
865 memory_object_size_t length,
866 __unused vm_sync_t sync_flags)
867 {
868 apple_protect_pager_t pager;
869
870 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %p\n", mem_obj));
871
872 pager = apple_protect_pager_lookup(mem_obj);
873
874 memory_object_synchronize_completed(pager->pager_control,
875 offset, length);
876
877 return KERN_SUCCESS;
878 }
879
880 /*
881 * apple_protect_pager_map()
882 *
883 * This allows VM to let us, the EMM, know that this memory object
884 * is currently mapped one or more times. This is called by VM each time
885 * the memory object gets mapped and we take one extra reference on the
886 * memory object to account for all its mappings.
887 */
888 kern_return_t
889 apple_protect_pager_map(
890 memory_object_t mem_obj,
891 __unused vm_prot_t prot)
892 {
893 apple_protect_pager_t pager;
894
895 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj));
896
897 pager = apple_protect_pager_lookup(mem_obj);
898
899 lck_mtx_lock(&apple_protect_pager_lock);
900 assert(pager->is_ready);
901 assert(pager->ref_count > 0); /* pager is alive */
902 if (pager->is_mapped == FALSE) {
903 /*
904 * First mapping of this pager: take an extra reference
905 * that will remain until all the mappings of this pager
906 * are removed.
907 */
908 pager->is_mapped = TRUE;
909 pager->ref_count++;
910 apple_protect_pager_count_mapped++;
911 }
912 lck_mtx_unlock(&apple_protect_pager_lock);
913
914 return KERN_SUCCESS;
915 }
916
917 /*
918 * apple_protect_pager_last_unmap()
919 *
920 * This is called by VM when this memory object is no longer mapped anywhere.
921 */
922 kern_return_t
923 apple_protect_pager_last_unmap(
924 memory_object_t mem_obj)
925 {
926 apple_protect_pager_t pager;
927 int count_unmapped;
928
929 PAGER_DEBUG(PAGER_ALL,
930 ("apple_protect_pager_last_unmap: %p\n", mem_obj));
931
932 pager = apple_protect_pager_lookup(mem_obj);
933
934 lck_mtx_lock(&apple_protect_pager_lock);
935 if (pager->is_mapped) {
936 /*
937 * All the mappings are gone, so let go of the one extra
938 * reference that represents all the mappings of this pager.
939 */
940 apple_protect_pager_count_mapped--;
941 count_unmapped = (apple_protect_pager_count -
942 apple_protect_pager_count_mapped);
943 if (count_unmapped > apple_protect_pager_count_unmapped_max) {
944 apple_protect_pager_count_unmapped_max = count_unmapped;
945 }
946 pager->is_mapped = FALSE;
947 apple_protect_pager_deallocate_internal(pager, TRUE);
948 /* caution: deallocate_internal() released the lock ! */
949 } else {
950 lck_mtx_unlock(&apple_protect_pager_lock);
951 }
952
953 return KERN_SUCCESS;
954 }
955
956
957 /*
958 *
959 */
960 apple_protect_pager_t
961 apple_protect_pager_lookup(
962 memory_object_t mem_obj)
963 {
964 apple_protect_pager_t pager;
965
966 pager = (apple_protect_pager_t) mem_obj;
967 assert(pager->pager_ops == &apple_protect_pager_ops);
968 assert(pager->ref_count > 0);
969 return pager;
970 }
971
972 apple_protect_pager_t
973 apple_protect_pager_create(
974 vm_object_t backing_object,
975 struct pager_crypt_info *crypt_info)
976 {
977 apple_protect_pager_t pager, pager2;
978 memory_object_control_t control;
979 kern_return_t kr;
980
981 pager = (apple_protect_pager_t) kalloc(sizeof (*pager));
982 if (pager == APPLE_PROTECT_PAGER_NULL) {
983 return APPLE_PROTECT_PAGER_NULL;
984 }
985
986 /*
987 * The vm_map call takes both named entry ports and raw memory
988 * objects in the same parameter. We need to make sure that
989 * vm_map does not see this object as a named entry port. So,
990 * we reserve the first word in the object for a fake ip_kotype
991 * setting - that will tell vm_map to use it as a memory object.
992 */
993 pager->pager_ops = &apple_protect_pager_ops;
994 pager->pager_ikot = IKOT_MEMORY_OBJECT;
995 pager->is_ready = FALSE;/* not ready until it has a "name" */
996 pager->ref_count = 2; /* existence + setup reference */
997 pager->is_mapped = FALSE;
998 pager->pager_control = MEMORY_OBJECT_CONTROL_NULL;
999 pager->backing_object = backing_object;
1000 pager->crypt = *crypt_info;
1001
1002 vm_object_reference(backing_object);
1003
1004 lck_mtx_lock(&apple_protect_pager_lock);
1005 /* see if anyone raced us to create a pager for the same object */
1006 queue_iterate(&apple_protect_pager_queue,
1007 pager2,
1008 apple_protect_pager_t,
1009 pager_queue) {
1010 if (pager2->backing_object == backing_object) {
1011 break;
1012 }
1013 }
1014 if (! queue_end(&apple_protect_pager_queue,
1015 (queue_entry_t) pager2)) {
1016 /* while we hold the lock, transfer our setup ref to winner */
1017 pager2->ref_count++;
1018 /* we lost the race, down with the loser... */
1019 lck_mtx_unlock(&apple_protect_pager_lock);
1020 vm_object_deallocate(pager->backing_object);
1021 pager->backing_object = VM_OBJECT_NULL;
1022 kfree(pager, sizeof (*pager));
1023 /* ... and go with the winner */
1024 pager = pager2;
1025 /* let the winner make sure the pager gets ready */
1026 return pager;
1027 }
1028
1029 /* enter new pager at the head of our list of pagers */
1030 queue_enter_first(&apple_protect_pager_queue,
1031 pager,
1032 apple_protect_pager_t,
1033 pager_queue);
1034 apple_protect_pager_count++;
1035 if (apple_protect_pager_count > apple_protect_pager_count_max) {
1036 apple_protect_pager_count_max = apple_protect_pager_count;
1037 }
1038 lck_mtx_unlock(&apple_protect_pager_lock);
1039
1040 kr = memory_object_create_named((memory_object_t) pager,
1041 0,
1042 &control);
1043 assert(kr == KERN_SUCCESS);
1044
1045 lck_mtx_lock(&apple_protect_pager_lock);
1046 /* the new pager is now ready to be used */
1047 pager->is_ready = TRUE;
1048 lck_mtx_unlock(&apple_protect_pager_lock);
1049
1050 /* wakeup anyone waiting for this pager to be ready */
1051 thread_wakeup(&pager->is_ready);
1052
1053 return pager;
1054 }
1055
1056 /*
1057 * apple_protect_pager_setup()
1058 *
1059 * Provide the caller with a memory object backed by the provided
1060 * "backing_object" VM object. If such a memory object already exists,
1061 * re-use it, otherwise create a new memory object.
1062 */
1063 memory_object_t
1064 apple_protect_pager_setup(
1065 vm_object_t backing_object,
1066 struct pager_crypt_info *crypt_info)
1067 {
1068 apple_protect_pager_t pager;
1069
1070 lck_mtx_lock(&apple_protect_pager_lock);
1071
1072 queue_iterate(&apple_protect_pager_queue,
1073 pager,
1074 apple_protect_pager_t,
1075 pager_queue) {
1076 if (pager->backing_object == backing_object) {
1077 /* For the same object we must always use the same protection options */
1078 if (!((pager->crypt.page_decrypt == crypt_info->page_decrypt) &&
1079 (pager->crypt.crypt_ops == crypt_info->crypt_ops) )) {
1080 lck_mtx_unlock(&apple_protect_pager_lock);
1081 return MEMORY_OBJECT_NULL;
1082 }
1083 break;
1084 }
1085 }
1086 if (queue_end(&apple_protect_pager_queue,
1087 (queue_entry_t) pager)) {
1088 /* no existing pager for this backing object */
1089 pager = APPLE_PROTECT_PAGER_NULL;
1090 } else {
1091 /* make sure pager doesn't disappear */
1092 pager->ref_count++;
1093 }
1094
1095 lck_mtx_unlock(&apple_protect_pager_lock);
1096
1097 if (pager == APPLE_PROTECT_PAGER_NULL) {
1098 pager = apple_protect_pager_create(backing_object, crypt_info);
1099 if (pager == APPLE_PROTECT_PAGER_NULL) {
1100 return MEMORY_OBJECT_NULL;
1101 }
1102 }
1103
1104 lck_mtx_lock(&apple_protect_pager_lock);
1105 while (!pager->is_ready) {
1106 lck_mtx_sleep(&apple_protect_pager_lock,
1107 LCK_SLEEP_DEFAULT,
1108 &pager->is_ready,
1109 THREAD_UNINT);
1110 }
1111 lck_mtx_unlock(&apple_protect_pager_lock);
1112
1113 return (memory_object_t) pager;
1114 }
1115
1116 void
1117 apple_protect_pager_trim(void)
1118 {
1119 apple_protect_pager_t pager, prev_pager;
1120 queue_head_t trim_queue;
1121 int num_trim;
1122 int count_unmapped;
1123
1124 lck_mtx_lock(&apple_protect_pager_lock);
1125
1126 /*
1127 * We have too many pagers, try and trim some unused ones,
1128 * starting with the oldest pager at the end of the queue.
1129 */
1130 queue_init(&trim_queue);
1131 num_trim = 0;
1132
1133 for (pager = (apple_protect_pager_t)
1134 queue_last(&apple_protect_pager_queue);
1135 !queue_end(&apple_protect_pager_queue,
1136 (queue_entry_t) pager);
1137 pager = prev_pager) {
1138 /* get prev elt before we dequeue */
1139 prev_pager = (apple_protect_pager_t)
1140 queue_prev(&pager->pager_queue);
1141
1142 if (pager->ref_count == 2 &&
1143 pager->is_ready &&
1144 !pager->is_mapped) {
1145 /* this pager can be trimmed */
1146 num_trim++;
1147 /* remove this pager from the main list ... */
1148 apple_protect_pager_dequeue(pager);
1149 /* ... and add it to our trim queue */
1150 queue_enter_first(&trim_queue,
1151 pager,
1152 apple_protect_pager_t,
1153 pager_queue);
1154
1155 count_unmapped = (apple_protect_pager_count -
1156 apple_protect_pager_count_mapped);
1157 if (count_unmapped <= apple_protect_pager_cache_limit) {
1158 /* we have enough pagers to trim */
1159 break;
1160 }
1161 }
1162 }
1163 if (num_trim > apple_protect_pager_num_trim_max) {
1164 apple_protect_pager_num_trim_max = num_trim;
1165 }
1166 apple_protect_pager_num_trim_total += num_trim;
1167
1168 lck_mtx_unlock(&apple_protect_pager_lock);
1169
1170 /* terminate the trimmed pagers */
1171 while (!queue_empty(&trim_queue)) {
1172 queue_remove_first(&trim_queue,
1173 pager,
1174 apple_protect_pager_t,
1175 pager_queue);
1176 pager->pager_queue.next = NULL;
1177 pager->pager_queue.prev = NULL;
1178 assert(pager->ref_count == 2);
1179 /*
1180 * We can't call deallocate_internal() because the pager
1181 * has already been dequeued, but we still need to remove
1182 * a reference.
1183 */
1184 pager->ref_count--;
1185 apple_protect_pager_terminate_internal(pager);
1186 }
1187 }
mirror of XNU