]> git.saurik.com Git - apple/xnu.git/blob - osfmk/vm/vm_apple_protect.c
projects / apple / xnu.git / blob
? search:


c6da62e27d51f826dbd99328cccb8a720315d5cb
[apple/xnu.git] / osfmk / vm / vm_apple_protect.c
1 /*
2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/errno.h>
30
31 #include <mach/mach_types.h>
32 #include <mach/mach_traps.h>
33 #include <mach/host_priv.h>
34 #include <mach/kern_return.h>
35 #include <mach/memory_object_control.h>
36 #include <mach/memory_object_types.h>
37 #include <mach/port.h>
38 #include <mach/policy.h>
39 #include <mach/upl.h>
40 #include <mach/thread_act.h>
41 #include <mach/mach_vm.h>
42
43 #include <kern/host.h>
44 #include <kern/kalloc.h>
45 #include <kern/page_decrypt.h>
46 #include <kern/queue.h>
47 #include <kern/thread.h>
48
49 #include <ipc/ipc_port.h>
50 #include <ipc/ipc_space.h>
51
52 #include <default_pager/default_pager_types.h>
53 #include <default_pager/default_pager_object_server.h>
54
55 #include <vm/vm_fault.h>
56 #include <vm/vm_map.h>
57 #include <vm/vm_pageout.h>
58 #include <vm/memory_object.h>
59 #include <vm/vm_pageout.h>
60 #include <vm/vm_protos.h>
61
62
63 /*
64 * APPLE PROTECT MEMORY PAGER
65 *
66 * This external memory manager (EMM) handles memory from the encrypted
67 * sections of some executables protected by the DSMOS kernel extension.
68 *
69 * It mostly handles page-in requests (from memory_object_data_request()) by
70 * getting the encrypted data from its backing VM object, itself backed by
71 * the encrypted file, decrypting it and providing it to VM.
72 *
73 * The decrypted pages will never be dirtied, so the memory manager doesn't
74 * need to handle page-out requests (from memory_object_data_return()). The
75 * pages need to be mapped copy-on-write, so that the originals stay clean.
76 *
77 * We don't expect to have to handle a large number of apple-protected
78 * binaries, so the data structures are very simple (simple linked list)
79 * for now.
80 */
81
82 /* forward declarations */
83 void apple_protect_pager_reference(memory_object_t mem_obj);
84 void apple_protect_pager_deallocate(memory_object_t mem_obj);
85 kern_return_t apple_protect_pager_init(memory_object_t mem_obj,
86 memory_object_control_t control,
87 memory_object_cluster_size_t pg_size);
88 kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj);
89 kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj,
90 memory_object_offset_t offset,
91 memory_object_cluster_size_t length,
92 vm_prot_t protection_required,
93 memory_object_fault_info_t fault_info);
94 kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj,
95 memory_object_offset_t offset,
96 memory_object_cluster_size_t data_cnt,
97 memory_object_offset_t *resid_offset,
98 int *io_error,
99 boolean_t dirty,
100 boolean_t kernel_copy,
101 int upl_flags);
102 kern_return_t apple_protect_pager_data_initialize(memory_object_t mem_obj,
103 memory_object_offset_t offset,
104 memory_object_cluster_size_t data_cnt);
105 kern_return_t apple_protect_pager_data_unlock(memory_object_t mem_obj,
106 memory_object_offset_t offset,
107 memory_object_size_t size,
108 vm_prot_t desired_access);
109 kern_return_t apple_protect_pager_synchronize(memory_object_t mem_obj,
110 memory_object_offset_t offset,
111 memory_object_size_t length,
112 vm_sync_t sync_flags);
113 kern_return_t apple_protect_pager_map(memory_object_t mem_obj,
114 vm_prot_t prot);
115 kern_return_t apple_protect_pager_last_unmap(memory_object_t mem_obj);
116
117 /*
118 * Vector of VM operations for this EMM.
119 * These routines are invoked by VM via the memory_object_*() interfaces.
120 */
121 const struct memory_object_pager_ops apple_protect_pager_ops = {
122 apple_protect_pager_reference,
123 apple_protect_pager_deallocate,
124 apple_protect_pager_init,
125 apple_protect_pager_terminate,
126 apple_protect_pager_data_request,
127 apple_protect_pager_data_return,
128 apple_protect_pager_data_initialize,
129 apple_protect_pager_data_unlock,
130 apple_protect_pager_synchronize,
131 apple_protect_pager_map,
132 apple_protect_pager_last_unmap,
133 NULL, /* data_reclaim */
134 "apple protect pager"
135 };
136
137 /*
138 * The "apple_protect_pager" describes a memory object backed by
139 * the "apple protect" EMM.
140 */
141 typedef struct apple_protect_pager {
142 struct ipc_object_header pager_header; /* fake ip_kotype() */
143 memory_object_pager_ops_t pager_ops; /* == &apple_protect_pager_ops */
144 queue_chain_t pager_queue; /* next & prev pagers */
145 unsigned int ref_count; /* reference count */
146 boolean_t is_ready; /* is this pager ready ? */
147 boolean_t is_mapped; /* is this mem_obj mapped ? */
148 memory_object_control_t pager_control; /* mem object control handle */
149 vm_object_t backing_object; /* VM obj w/ encrypted data */
150 struct pager_crypt_info crypt;
151 } *apple_protect_pager_t;
152 #define APPLE_PROTECT_PAGER_NULL ((apple_protect_pager_t) NULL)
153 #define pager_ikot pager_header.io_bits
154
155 /*
156 * List of memory objects managed by this EMM.
157 * The list is protected by the "apple_protect_pager_lock" lock.
158 */
159 int apple_protect_pager_count = 0; /* number of pagers */
160 int apple_protect_pager_count_mapped = 0; /* number of unmapped pagers */
161 queue_head_t apple_protect_pager_queue;
162 decl_lck_mtx_data(,apple_protect_pager_lock)
163
164 /*
165 * Maximum number of unmapped pagers we're willing to keep around.
166 */
167 int apple_protect_pager_cache_limit = 10;
168
169 /*
170 * Statistics & counters.
171 */
172 int apple_protect_pager_count_max = 0;
173 int apple_protect_pager_count_unmapped_max = 0;
174 int apple_protect_pager_num_trim_max = 0;
175 int apple_protect_pager_num_trim_total = 0;
176
177
178 lck_grp_t apple_protect_pager_lck_grp;
179 lck_grp_attr_t apple_protect_pager_lck_grp_attr;
180 lck_attr_t apple_protect_pager_lck_attr;
181
182
183 /* internal prototypes */
184 apple_protect_pager_t apple_protect_pager_create(vm_object_t backing_object,
185 struct pager_crypt_info *crypt_info);
186 apple_protect_pager_t apple_protect_pager_lookup(memory_object_t mem_obj);
187 void apple_protect_pager_dequeue(apple_protect_pager_t pager);
188 void apple_protect_pager_deallocate_internal(apple_protect_pager_t pager,
189 boolean_t locked);
190 void apple_protect_pager_terminate_internal(apple_protect_pager_t pager);
191 void apple_protect_pager_trim(void);
192
193
194 #if DEBUG
195 int apple_protect_pagerdebug = 0;
196 #define PAGER_ALL 0xffffffff
197 #define PAGER_INIT 0x00000001
198 #define PAGER_PAGEIN 0x00000002
199
200 #define PAGER_DEBUG(LEVEL, A) \
201 MACRO_BEGIN \
202 if ((apple_protect_pagerdebug & LEVEL)==LEVEL) { \
203 printf A; \
204 } \
205 MACRO_END
206 #else
207 #define PAGER_DEBUG(LEVEL, A)
208 #endif
209
210
211 void
212 apple_protect_pager_bootstrap(void)
213 {
214 lck_grp_attr_setdefault(&apple_protect_pager_lck_grp_attr);
215 lck_grp_init(&apple_protect_pager_lck_grp, "apple_protect", &apple_protect_pager_lck_grp_attr);
216 lck_attr_setdefault(&apple_protect_pager_lck_attr);
217 lck_mtx_init(&apple_protect_pager_lock, &apple_protect_pager_lck_grp, &apple_protect_pager_lck_attr);
218 queue_init(&apple_protect_pager_queue);
219 }
220
221 /*
222 * apple_protect_pager_init()
223 *
224 * Initialize the memory object and makes it ready to be used and mapped.
225 */
226 kern_return_t
227 apple_protect_pager_init(
228 memory_object_t mem_obj,
229 memory_object_control_t control,
230 #if !DEBUG
231 __unused
232 #endif
233 memory_object_cluster_size_t pg_size)
234 {
235 apple_protect_pager_t pager;
236 kern_return_t kr;
237 memory_object_attr_info_data_t attributes;
238
239 PAGER_DEBUG(PAGER_ALL,
240 ("apple_protect_pager_init: %p, %p, %x\n",
241 mem_obj, control, pg_size));
242
243 if (control == MEMORY_OBJECT_CONTROL_NULL)
244 return KERN_INVALID_ARGUMENT;
245
246 pager = apple_protect_pager_lookup(mem_obj);
247
248 memory_object_control_reference(control);
249
250 pager->pager_control = control;
251
252 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
253 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
254 attributes.cluster_size = (1 << (PAGE_SHIFT));
255 attributes.may_cache_object = FALSE;
256 attributes.temporary = TRUE;
257
258 kr = memory_object_change_attributes(
259 control,
260 MEMORY_OBJECT_ATTRIBUTE_INFO,
261 (memory_object_info_t) &attributes,
262 MEMORY_OBJECT_ATTR_INFO_COUNT);
263 if (kr != KERN_SUCCESS)
264 panic("apple_protect_pager_init: "
265 "memory_object_change_attributes() failed");
266
267 return KERN_SUCCESS;
268 }
269
270 /*
271 * apple_protect_data_return()
272 *
273 * Handles page-out requests from VM. This should never happen since
274 * the pages provided by this EMM are not supposed to be dirty or dirtied
275 * and VM should simply discard the contents and reclaim the pages if it
276 * needs to.
277 */
278 kern_return_t
279 apple_protect_pager_data_return(
280 __unused memory_object_t mem_obj,
281 __unused memory_object_offset_t offset,
282 __unused memory_object_cluster_size_t data_cnt,
283 __unused memory_object_offset_t *resid_offset,
284 __unused int *io_error,
285 __unused boolean_t dirty,
286 __unused boolean_t kernel_copy,
287 __unused int upl_flags)
288 {
289 panic("apple_protect_pager_data_return: should never get called");
290 return KERN_FAILURE;
291 }
292
293 kern_return_t
294 apple_protect_pager_data_initialize(
295 __unused memory_object_t mem_obj,
296 __unused memory_object_offset_t offset,
297 __unused memory_object_cluster_size_t data_cnt)
298 {
299 panic("apple_protect_pager_data_initialize: should never get called");
300 return KERN_FAILURE;
301 }
302
303 kern_return_t
304 apple_protect_pager_data_unlock(
305 __unused memory_object_t mem_obj,
306 __unused memory_object_offset_t offset,
307 __unused memory_object_size_t size,
308 __unused vm_prot_t desired_access)
309 {
310 return KERN_FAILURE;
311 }
312
313 /*
314 * apple_protect_pager_data_request()
315 *
316 * Handles page-in requests from VM.
317 */
318 kern_return_t
319 apple_protect_pager_data_request(
320 memory_object_t mem_obj,
321 memory_object_offset_t offset,
322 memory_object_cluster_size_t length,
323 #if !DEBUG
324 __unused
325 #endif
326 vm_prot_t protection_required,
327 memory_object_fault_info_t mo_fault_info)
328 {
329 apple_protect_pager_t pager;
330 memory_object_control_t mo_control;
331 upl_t upl;
332 int upl_flags;
333 upl_size_t upl_size;
334 upl_page_info_t *upl_pl;
335 unsigned int pl_count;
336 vm_object_t src_object, dst_object;
337 kern_return_t kr, retval;
338 vm_map_offset_t kernel_mapping;
339 vm_offset_t src_vaddr, dst_vaddr;
340 vm_offset_t cur_offset;
341 vm_map_entry_t map_entry;
342 kern_return_t error_code;
343 vm_prot_t prot;
344 vm_page_t src_page, top_page;
345 int interruptible;
346 struct vm_object_fault_info fault_info;
347 int ret;
348
349 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required));
350
351 retval = KERN_SUCCESS;
352 src_object = VM_OBJECT_NULL;
353 kernel_mapping = 0;
354 upl = NULL;
355 upl_pl = NULL;
356 fault_info = *((struct vm_object_fault_info *) mo_fault_info);
357 fault_info.stealth = TRUE;
358 fault_info.io_sync = FALSE;
359 fault_info.mark_zf_absent = FALSE;
360 fault_info.batch_pmap_op = FALSE;
361 interruptible = fault_info.interruptible;
362
363 pager = apple_protect_pager_lookup(mem_obj);
364 assert(pager->is_ready);
365 assert(pager->ref_count > 1); /* pager is alive and mapped */
366
367 PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
368
369 /*
370 * Gather in a UPL all the VM pages requested by VM.
371 */
372 mo_control = pager->pager_control;
373
374 upl_size = length;
375 upl_flags =
376 UPL_RET_ONLY_ABSENT |
377 UPL_SET_LITE |
378 UPL_NO_SYNC |
379 UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */
380 UPL_SET_INTERNAL;
381 pl_count = 0;
382 kr = memory_object_upl_request(mo_control,
383 offset, upl_size,
384 &upl, NULL, NULL, upl_flags);
385 if (kr != KERN_SUCCESS) {
386 retval = kr;
387 goto done;
388 }
389 dst_object = mo_control->moc_object;
390 assert(dst_object != VM_OBJECT_NULL);
391
392
393 /*
394 * Reserve 2 virtual pages in the kernel address space to map each
395 * source and destination physical pages when it's their turn to
396 * be processed.
397 */
398 vm_object_reference(kernel_object); /* ref. for mapping */
399 kr = vm_map_find_space(kernel_map,
400 &kernel_mapping,
401 2 * PAGE_SIZE_64,
402 0,
403 0,
404 &map_entry);
405 if (kr != KERN_SUCCESS) {
406 vm_object_deallocate(kernel_object);
407 retval = kr;
408 goto done;
409 }
410 map_entry->object.vm_object = kernel_object;
411 map_entry->offset = kernel_mapping;
412 vm_map_unlock(kernel_map);
413 src_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping);
414 dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping + PAGE_SIZE_64);
415
416 /*
417 * We'll map the encrypted data in the kernel address space from the
418 * backing VM object (itself backed by the encrypted file via
419 * the vnode pager).
420 */
421 src_object = pager->backing_object;
422 assert(src_object != VM_OBJECT_NULL);
423 vm_object_reference(src_object); /* to keep the source object alive */
424
425 /*
426 * Fill in the contents of the pages requested by VM.
427 */
428 upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
429 pl_count = length / PAGE_SIZE;
430 for (cur_offset = 0;
431 retval == KERN_SUCCESS && cur_offset < length;
432 cur_offset += PAGE_SIZE) {
433 ppnum_t dst_pnum;
434
435 if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) {
436 /* this page is not in the UPL: skip it */
437 continue;
438 }
439
440 /*
441 * Map the source (encrypted) page in the kernel's
442 * virtual address space.
443 * We already hold a reference on the src_object.
444 */
445 retry_src_fault:
446 vm_object_lock(src_object);
447 vm_object_paging_begin(src_object);
448 error_code = 0;
449 prot = VM_PROT_READ;
450 src_page = VM_PAGE_NULL;
451 kr = vm_fault_page(src_object,
452 offset + cur_offset,
453 VM_PROT_READ,
454 FALSE,
455 FALSE, /* src_page not looked up */
456 &prot,
457 &src_page,
458 &top_page,
459 NULL,
460 &error_code,
461 FALSE,
462 FALSE,
463 &fault_info);
464 switch (kr) {
465 case VM_FAULT_SUCCESS:
466 break;
467 case VM_FAULT_RETRY:
468 goto retry_src_fault;
469 case VM_FAULT_MEMORY_SHORTAGE:
470 if (vm_page_wait(interruptible)) {
471 goto retry_src_fault;
472 }
473 /* fall thru */
474 case VM_FAULT_INTERRUPTED:
475 retval = MACH_SEND_INTERRUPTED;
476 goto done;
477 case VM_FAULT_SUCCESS_NO_VM_PAGE:
478 /* success but no VM page: fail */
479 vm_object_paging_end(src_object);
480 vm_object_unlock(src_object);
481 /*FALLTHROUGH*/
482 case VM_FAULT_MEMORY_ERROR:
483 /* the page is not there ! */
484 if (error_code) {
485 retval = error_code;
486 } else {
487 retval = KERN_MEMORY_ERROR;
488 }
489 goto done;
490 default:
491 panic("apple_protect_pager_data_request: "
492 "vm_fault_page() unexpected error 0x%x\n",
493 kr);
494 }
495 assert(src_page != VM_PAGE_NULL);
496 assert(src_page->busy);
497
498 if (!src_page->active &&
499 !src_page->inactive &&
500 !src_page->throttled) {
501 vm_page_lockspin_queues();
502 if (!src_page->active &&
503 !src_page->inactive &&
504 !src_page->throttled) {
505 vm_page_deactivate(src_page);
506 }
507 vm_page_unlock_queues();
508 }
509
510 /*
511 * Establish an explicit mapping of the source
512 * physical page.
513 */
514 pmap_enter(kernel_pmap,
515 kernel_mapping,
516 src_page->phys_page,
517 VM_PROT_READ,
518 VM_PROT_NONE,
519 0,
520 TRUE);
521 /*
522 * Establish an explicit pmap mapping of the destination
523 * physical page.
524 * We can't do a regular VM mapping because the VM page
525 * is "busy".
526 */
527 dst_pnum = (ppnum_t)
528 upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE));
529 assert(dst_pnum != 0);
530 pmap_enter(kernel_pmap,
531 kernel_mapping + PAGE_SIZE_64,
532 dst_pnum,
533 VM_PROT_READ | VM_PROT_WRITE,
534 VM_PROT_NONE,
535 0,
536 TRUE);
537
538 /*
539 * Decrypt the encrypted contents of the source page
540 * into the destination page.
541 */
542 ret = pager->crypt.page_decrypt((const void *) src_vaddr,
543 (void *) dst_vaddr,
544 offset+cur_offset,
545 pager->crypt.crypt_ops);
546 if (ret) {
547 /*
548 * Decryption failed. Abort the fault.
549 */
550 retval = KERN_ABORTED;
551 } else {
552 /*
553 * Validate the original page...
554 */
555 if (src_page->object->code_signed) {
556 vm_page_validate_cs_mapped(
557 src_page,
558 (const void *) src_vaddr);
559 }
560 /*
561 * ... and transfer the results to the destination page.
562 */
563 UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE,
564 src_page->cs_validated);
565 UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE,
566 src_page->cs_tainted);
567 }
568
569 /*
570 * Remove the pmap mapping of the source and destination pages
571 * in the kernel.
572 */
573 pmap_remove(kernel_pmap,
574 (addr64_t) kernel_mapping,
575 (addr64_t) (kernel_mapping + (2 * PAGE_SIZE_64)));
576
577 /*
578 * Cleanup the result of vm_fault_page() of the source page.
579 */
580 PAGE_WAKEUP_DONE(src_page);
581 vm_object_paging_end(src_page->object);
582 vm_object_unlock(src_page->object);
583 if (top_page != VM_PAGE_NULL) {
584 vm_object_t top_object;
585
586 top_object = top_page->object;
587 vm_object_lock(top_object);
588 VM_PAGE_FREE(top_page);
589 vm_object_paging_end(top_object);
590 vm_object_unlock(top_object);
591 }
592 }
593
594 done:
595 if (upl != NULL) {
596 /* clean up the UPL */
597
598 /*
599 * The pages are currently dirty because we've just been
600 * writing on them, but as far as we're concerned, they're
601 * clean since they contain their "original" contents as
602 * provided by us, the pager.
603 * Tell the UPL to mark them "clean".
604 */
605 upl_clear_dirty(upl, TRUE);
606
607 /* abort or commit the UPL */
608 if (retval != KERN_SUCCESS) {
609 upl_abort(upl, 0);
610 if (retval == KERN_ABORTED) {
611 wait_result_t wait_result;
612
613 /*
614 * We aborted the fault and did not provide
615 * any contents for the requested pages but
616 * the pages themselves are not invalid, so
617 * let's return success and let the caller
618 * retry the fault, in case it might succeed
619 * later (when the decryption code is up and
620 * running in the kernel, for example).
621 */
622 retval = KERN_SUCCESS;
623 /*
624 * Wait a little bit first to avoid using
625 * too much CPU time retrying and failing
626 * the same fault over and over again.
627 */
628 wait_result = assert_wait_timeout(
629 (event_t) apple_protect_pager_data_request,
630 THREAD_UNINT,
631 10000, /* 10ms */
632 NSEC_PER_USEC);
633 assert(wait_result == THREAD_WAITING);
634 wait_result = thread_block(THREAD_CONTINUE_NULL);
635 assert(wait_result == THREAD_TIMED_OUT);
636 }
637 } else {
638 boolean_t empty;
639 upl_commit_range(upl, 0, upl->size,
640 UPL_COMMIT_CS_VALIDATED,
641 upl_pl, pl_count, &empty);
642 }
643
644 /* and deallocate the UPL */
645 upl_deallocate(upl);
646 upl = NULL;
647 }
648 if (kernel_mapping != 0) {
649 /* clean up the mapping of the source and destination pages */
650 kr = vm_map_remove(kernel_map,
651 kernel_mapping,
652 kernel_mapping + (2 * PAGE_SIZE_64),
653 VM_MAP_NO_FLAGS);
654 assert(kr == KERN_SUCCESS);
655 kernel_mapping = 0;
656 src_vaddr = 0;
657 dst_vaddr = 0;
658 }
659 if (src_object != VM_OBJECT_NULL) {
660 vm_object_deallocate(src_object);
661 }
662
663 return retval;
664 }
665
666 /*
667 * apple_protect_pager_reference()
668 *
669 * Get a reference on this memory object.
670 * For external usage only. Assumes that the initial reference count is not 0,
671 * i.e one should not "revive" a dead pager this way.
672 */
673 void
674 apple_protect_pager_reference(
675 memory_object_t mem_obj)
676 {
677 apple_protect_pager_t pager;
678
679 pager = apple_protect_pager_lookup(mem_obj);
680
681 lck_mtx_lock(&apple_protect_pager_lock);
682 assert(pager->ref_count > 0);
683 pager->ref_count++;
684 lck_mtx_unlock(&apple_protect_pager_lock);
685 }
686
687
688 /*
689 * apple_protect_pager_dequeue:
690 *
691 * Removes a pager from the list of pagers.
692 *
693 * The caller must hold "apple_protect_pager_lock".
694 */
695 void
696 apple_protect_pager_dequeue(
697 apple_protect_pager_t pager)
698 {
699 assert(!pager->is_mapped);
700
701 queue_remove(&apple_protect_pager_queue,
702 pager,
703 apple_protect_pager_t,
704 pager_queue);
705 pager->pager_queue.next = NULL;
706 pager->pager_queue.prev = NULL;
707
708 apple_protect_pager_count--;
709 }
710
711 /*
712 * apple_protect_pager_terminate_internal:
713 *
714 * Trigger the asynchronous termination of the memory object associated
715 * with this pager.
716 * When the memory object is terminated, there will be one more call
717 * to memory_object_deallocate() (i.e. apple_protect_pager_deallocate())
718 * to finish the clean up.
719 *
720 * "apple_protect_pager_lock" should not be held by the caller.
721 * We don't need the lock because the pager has already been removed from
722 * the pagers' list and is now ours exclusively.
723 */
724 void
725 apple_protect_pager_terminate_internal(
726 apple_protect_pager_t pager)
727 {
728 assert(pager->is_ready);
729 assert(!pager->is_mapped);
730
731 if (pager->backing_object != VM_OBJECT_NULL) {
732 vm_object_deallocate(pager->backing_object);
733 pager->backing_object = VM_OBJECT_NULL;
734 }
735
736 /* deallocate any crypt module data */
737 if(pager->crypt.crypt_end)
738 pager->crypt.crypt_end(pager->crypt.crypt_ops);
739
740 /* trigger the destruction of the memory object */
741 memory_object_destroy(pager->pager_control, 0);
742 }
743
744 /*
745 * apple_protect_pager_deallocate_internal()
746 *
747 * Release a reference on this pager and free it when the last
748 * reference goes away.
749 * Can be called with apple_protect_pager_lock held or not but always returns
750 * with it unlocked.
751 */
752 void
753 apple_protect_pager_deallocate_internal(
754 apple_protect_pager_t pager,
755 boolean_t locked)
756 {
757 boolean_t needs_trimming;
758 int count_unmapped;
759
760 if (! locked) {
761 lck_mtx_lock(&apple_protect_pager_lock);
762 }
763
764 count_unmapped = (apple_protect_pager_count -
765 apple_protect_pager_count_mapped);
766 if (count_unmapped > apple_protect_pager_cache_limit) {
767 /* we have too many unmapped pagers: trim some */
768 needs_trimming = TRUE;
769 } else {
770 needs_trimming = FALSE;
771 }
772
773 /* drop a reference on this pager */
774 pager->ref_count--;
775
776 if (pager->ref_count == 1) {
777 /*
778 * Only the "named" reference is left, which means that
779 * no one is really holding on to this pager anymore.
780 * Terminate it.
781 */
782 apple_protect_pager_dequeue(pager);
783 /* the pager is all ours: no need for the lock now */
784 lck_mtx_unlock(&apple_protect_pager_lock);
785 apple_protect_pager_terminate_internal(pager);
786 } else if (pager->ref_count == 0) {
787 /*
788 * Dropped the existence reference; the memory object has
789 * been terminated. Do some final cleanup and release the
790 * pager structure.
791 */
792 lck_mtx_unlock(&apple_protect_pager_lock);
793 if (pager->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
794 memory_object_control_deallocate(pager->pager_control);
795 pager->pager_control = MEMORY_OBJECT_CONTROL_NULL;
796 }
797 kfree(pager, sizeof (*pager));
798 pager = APPLE_PROTECT_PAGER_NULL;
799 } else {
800 /* there are still plenty of references: keep going... */
801 lck_mtx_unlock(&apple_protect_pager_lock);
802 }
803
804 if (needs_trimming) {
805 apple_protect_pager_trim();
806 }
807 /* caution: lock is not held on return... */
808 }
809
810 /*
811 * apple_protect_pager_deallocate()
812 *
813 * Release a reference on this pager and free it when the last
814 * reference goes away.
815 */
816 void
817 apple_protect_pager_deallocate(
818 memory_object_t mem_obj)
819 {
820 apple_protect_pager_t pager;
821
822 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj));
823 pager = apple_protect_pager_lookup(mem_obj);
824 apple_protect_pager_deallocate_internal(pager, FALSE);
825 }
826
827 /*
828 *
829 */
830 kern_return_t
831 apple_protect_pager_terminate(
832 #if !DEBUG
833 __unused
834 #endif
835 memory_object_t mem_obj)
836 {
837 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj));
838
839 return KERN_SUCCESS;
840 }
841
842 /*
843 *
844 */
845 kern_return_t
846 apple_protect_pager_synchronize(
847 memory_object_t mem_obj,
848 memory_object_offset_t offset,
849 memory_object_size_t length,
850 __unused vm_sync_t sync_flags)
851 {
852 apple_protect_pager_t pager;
853
854 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %p\n", mem_obj));
855
856 pager = apple_protect_pager_lookup(mem_obj);
857
858 memory_object_synchronize_completed(pager->pager_control,
859 offset, length);
860
861 return KERN_SUCCESS;
862 }
863
864 /*
865 * apple_protect_pager_map()
866 *
867 * This allows VM to let us, the EMM, know that this memory object
868 * is currently mapped one or more times. This is called by VM each time
869 * the memory object gets mapped and we take one extra reference on the
870 * memory object to account for all its mappings.
871 */
872 kern_return_t
873 apple_protect_pager_map(
874 memory_object_t mem_obj,
875 __unused vm_prot_t prot)
876 {
877 apple_protect_pager_t pager;
878
879 PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj));
880
881 pager = apple_protect_pager_lookup(mem_obj);
882
883 lck_mtx_lock(&apple_protect_pager_lock);
884 assert(pager->is_ready);
885 assert(pager->ref_count > 0); /* pager is alive */
886 if (pager->is_mapped == FALSE) {
887 /*
888 * First mapping of this pager: take an extra reference
889 * that will remain until all the mappings of this pager
890 * are removed.
891 */
892 pager->is_mapped = TRUE;
893 pager->ref_count++;
894 apple_protect_pager_count_mapped++;
895 }
896 lck_mtx_unlock(&apple_protect_pager_lock);
897
898 return KERN_SUCCESS;
899 }
900
901 /*
902 * apple_protect_pager_last_unmap()
903 *
904 * This is called by VM when this memory object is no longer mapped anywhere.
905 */
906 kern_return_t
907 apple_protect_pager_last_unmap(
908 memory_object_t mem_obj)
909 {
910 apple_protect_pager_t pager;
911 int count_unmapped;
912
913 PAGER_DEBUG(PAGER_ALL,
914 ("apple_protect_pager_last_unmap: %p\n", mem_obj));
915
916 pager = apple_protect_pager_lookup(mem_obj);
917
918 lck_mtx_lock(&apple_protect_pager_lock);
919 if (pager->is_mapped) {
920 /*
921 * All the mappings are gone, so let go of the one extra
922 * reference that represents all the mappings of this pager.
923 */
924 apple_protect_pager_count_mapped--;
925 count_unmapped = (apple_protect_pager_count -
926 apple_protect_pager_count_mapped);
927 if (count_unmapped > apple_protect_pager_count_unmapped_max) {
928 apple_protect_pager_count_unmapped_max = count_unmapped;
929 }
930 pager->is_mapped = FALSE;
931 apple_protect_pager_deallocate_internal(pager, TRUE);
932 /* caution: deallocate_internal() released the lock ! */
933 } else {
934 lck_mtx_unlock(&apple_protect_pager_lock);
935 }
936
937 return KERN_SUCCESS;
938 }
939
940
941 /*
942 *
943 */
944 apple_protect_pager_t
945 apple_protect_pager_lookup(
946 memory_object_t mem_obj)
947 {
948 apple_protect_pager_t pager;
949
950 pager = (apple_protect_pager_t) mem_obj;
951 assert(pager->pager_ops == &apple_protect_pager_ops);
952 assert(pager->ref_count > 0);
953 return pager;
954 }
955
956 apple_protect_pager_t
957 apple_protect_pager_create(
958 vm_object_t backing_object,
959 struct pager_crypt_info *crypt_info)
960 {
961 apple_protect_pager_t pager, pager2;
962 memory_object_control_t control;
963 kern_return_t kr;
964
965 pager = (apple_protect_pager_t) kalloc(sizeof (*pager));
966 if (pager == APPLE_PROTECT_PAGER_NULL) {
967 return APPLE_PROTECT_PAGER_NULL;
968 }
969
970 /*
971 * The vm_map call takes both named entry ports and raw memory
972 * objects in the same parameter. We need to make sure that
973 * vm_map does not see this object as a named entry port. So,
974 * we reserve the first word in the object for a fake ip_kotype
975 * setting - that will tell vm_map to use it as a memory object.
976 */
977 pager->pager_ops = &apple_protect_pager_ops;
978 pager->pager_ikot = IKOT_MEMORY_OBJECT;
979 pager->is_ready = FALSE;/* not ready until it has a "name" */
980 pager->ref_count = 2; /* existence + setup reference */
981 pager->is_mapped = FALSE;
982 pager->pager_control = MEMORY_OBJECT_CONTROL_NULL;
983 pager->backing_object = backing_object;
984 pager->crypt = *crypt_info;
985
986 vm_object_reference(backing_object);
987
988 lck_mtx_lock(&apple_protect_pager_lock);
989 /* see if anyone raced us to create a pager for the same object */
990 queue_iterate(&apple_protect_pager_queue,
991 pager2,
992 apple_protect_pager_t,
993 pager_queue) {
994 if (pager2->backing_object == backing_object) {
995 break;
996 }
997 }
998 if (! queue_end(&apple_protect_pager_queue,
999 (queue_entry_t) pager2)) {
1000 /* while we hold the lock, transfer our setup ref to winner */
1001 pager2->ref_count++;
1002 /* we lost the race, down with the loser... */
1003 lck_mtx_unlock(&apple_protect_pager_lock);
1004 vm_object_deallocate(pager->backing_object);
1005 pager->backing_object = VM_OBJECT_NULL;
1006 kfree(pager, sizeof (*pager));
1007 /* ... and go with the winner */
1008 pager = pager2;
1009 /* let the winner make sure the pager gets ready */
1010 return pager;
1011 }
1012
1013 /* enter new pager at the head of our list of pagers */
1014 queue_enter_first(&apple_protect_pager_queue,
1015 pager,
1016 apple_protect_pager_t,
1017 pager_queue);
1018 apple_protect_pager_count++;
1019 if (apple_protect_pager_count > apple_protect_pager_count_max) {
1020 apple_protect_pager_count_max = apple_protect_pager_count;
1021 }
1022 lck_mtx_unlock(&apple_protect_pager_lock);
1023
1024 kr = memory_object_create_named((memory_object_t) pager,
1025 0,
1026 &control);
1027 assert(kr == KERN_SUCCESS);
1028
1029 lck_mtx_lock(&apple_protect_pager_lock);
1030 /* the new pager is now ready to be used */
1031 pager->is_ready = TRUE;
1032 lck_mtx_unlock(&apple_protect_pager_lock);
1033
1034 /* wakeup anyone waiting for this pager to be ready */
1035 thread_wakeup(&pager->is_ready);
1036
1037 return pager;
1038 }
1039
1040 /*
1041 * apple_protect_pager_setup()
1042 *
1043 * Provide the caller with a memory object backed by the provided
1044 * "backing_object" VM object. If such a memory object already exists,
1045 * re-use it, otherwise create a new memory object.
1046 */
1047 memory_object_t
1048 apple_protect_pager_setup(
1049 vm_object_t backing_object,
1050 struct pager_crypt_info *crypt_info)
1051 {
1052 apple_protect_pager_t pager;
1053
1054 lck_mtx_lock(&apple_protect_pager_lock);
1055
1056 queue_iterate(&apple_protect_pager_queue,
1057 pager,
1058 apple_protect_pager_t,
1059 pager_queue) {
1060 if (pager->backing_object == backing_object) {
1061 /* For the same object we must always use the same protection options */
1062 if (!((pager->crypt.page_decrypt == crypt_info->page_decrypt) &&
1063 (pager->crypt.crypt_ops == crypt_info->crypt_ops) )) {
1064 lck_mtx_unlock(&apple_protect_pager_lock);
1065 return MEMORY_OBJECT_NULL;
1066 }
1067 break;
1068 }
1069 }
1070 if (queue_end(&apple_protect_pager_queue,
1071 (queue_entry_t) pager)) {
1072 /* no existing pager for this backing object */
1073 pager = APPLE_PROTECT_PAGER_NULL;
1074 } else {
1075 /* make sure pager doesn't disappear */
1076 pager->ref_count++;
1077 }
1078
1079 lck_mtx_unlock(&apple_protect_pager_lock);
1080
1081 if (pager == APPLE_PROTECT_PAGER_NULL) {
1082 pager = apple_protect_pager_create(backing_object, crypt_info);
1083 if (pager == APPLE_PROTECT_PAGER_NULL) {
1084 return MEMORY_OBJECT_NULL;
1085 }
1086 }
1087
1088 lck_mtx_lock(&apple_protect_pager_lock);
1089 while (!pager->is_ready) {
1090 lck_mtx_sleep(&apple_protect_pager_lock,
1091 LCK_SLEEP_DEFAULT,
1092 &pager->is_ready,
1093 THREAD_UNINT);
1094 }
1095 lck_mtx_unlock(&apple_protect_pager_lock);
1096
1097 return (memory_object_t) pager;
1098 }
1099
1100 void
1101 apple_protect_pager_trim(void)
1102 {
1103 apple_protect_pager_t pager, prev_pager;
1104 queue_head_t trim_queue;
1105 int num_trim;
1106 int count_unmapped;
1107
1108 lck_mtx_lock(&apple_protect_pager_lock);
1109
1110 /*
1111 * We have too many pagers, try and trim some unused ones,
1112 * starting with the oldest pager at the end of the queue.
1113 */
1114 queue_init(&trim_queue);
1115 num_trim = 0;
1116
1117 for (pager = (apple_protect_pager_t)
1118 queue_last(&apple_protect_pager_queue);
1119 !queue_end(&apple_protect_pager_queue,
1120 (queue_entry_t) pager);
1121 pager = prev_pager) {
1122 /* get prev elt before we dequeue */
1123 prev_pager = (apple_protect_pager_t)
1124 queue_prev(&pager->pager_queue);
1125
1126 if (pager->ref_count == 2 &&
1127 pager->is_ready &&
1128 !pager->is_mapped) {
1129 /* this pager can be trimmed */
1130 num_trim++;
1131 /* remove this pager from the main list ... */
1132 apple_protect_pager_dequeue(pager);
1133 /* ... and add it to our trim queue */
1134 queue_enter_first(&trim_queue,
1135 pager,
1136 apple_protect_pager_t,
1137 pager_queue);
1138
1139 count_unmapped = (apple_protect_pager_count -
1140 apple_protect_pager_count_mapped);
1141 if (count_unmapped <= apple_protect_pager_cache_limit) {
1142 /* we have enough pagers to trim */
1143 break;
1144 }
1145 }
1146 }
1147 if (num_trim > apple_protect_pager_num_trim_max) {
1148 apple_protect_pager_num_trim_max = num_trim;
1149 }
1150 apple_protect_pager_num_trim_total += num_trim;
1151
1152 lck_mtx_unlock(&apple_protect_pager_lock);
1153
1154 /* terminate the trimmed pagers */
1155 while (!queue_empty(&trim_queue)) {
1156 queue_remove_first(&trim_queue,
1157 pager,
1158 apple_protect_pager_t,
1159 pager_queue);
1160 pager->pager_queue.next = NULL;
1161 pager->pager_queue.prev = NULL;
1162 assert(pager->ref_count == 2);
1163 /*
1164 * We can't call deallocate_internal() because the pager
1165 * has already been dequeued, but we still need to remove
1166 * a reference.
1167 */
1168 pager->ref_count--;
1169 apple_protect_pager_terminate_internal(pager);
1170 }
1171 }
mirror of XNU