]> git.saurik.com Git - apple/xnu.git/blob - bsd/vm/vm_unix.c
projects / apple / xnu.git / blob
? search:


9421ee0f9655530125aeabfc02b06747f9b12bff
[apple/xnu.git] / bsd / vm / vm_unix.c
1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Mach Operating System
30 * Copyright (c) 1987 Carnegie-Mellon University
31 * All rights reserved. The CMU software License Agreement specifies
32 * the terms and conditions for use and redistribution.
33 */
34 /*
35 * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
36 * support for mandatory and extensible security protections. This notice
37 * is included in support of clause 2.2 (b) of the Apple Public License,
38 * Version 2.0.
39 */
40 #include <vm/vm_options.h>
41
42 #include <kern/task.h>
43 #include <kern/thread.h>
44 #include <kern/debug.h>
45 #include <kern/extmod_statistics.h>
46 #include <mach/mach_traps.h>
47 #include <mach/port.h>
48 #include <mach/sdt.h>
49 #include <mach/task.h>
50 #include <mach/task_access.h>
51 #include <mach/task_special_ports.h>
52 #include <mach/time_value.h>
53 #include <mach/vm_map.h>
54 #include <mach/vm_param.h>
55 #include <mach/vm_prot.h>
56 #include <machine/machine_routines.h>
57
58 #include <sys/file_internal.h>
59 #include <sys/param.h>
60 #include <sys/systm.h>
61 #include <sys/dir.h>
62 #include <sys/namei.h>
63 #include <sys/proc_internal.h>
64 #include <sys/kauth.h>
65 #include <sys/vm.h>
66 #include <sys/file.h>
67 #include <sys/vnode_internal.h>
68 #include <sys/mount.h>
69 #include <sys/xattr.h>
70 #include <sys/trace.h>
71 #include <sys/kernel.h>
72 #include <sys/ubc_internal.h>
73 #include <sys/user.h>
74 #include <sys/syslog.h>
75 #include <sys/stat.h>
76 #include <sys/sysproto.h>
77 #include <sys/mman.h>
78 #include <sys/sysctl.h>
79 #include <sys/cprotect.h>
80 #include <sys/kpi_socket.h>
81 #include <sys/kas_info.h>
82 #include <sys/socket.h>
83 #include <sys/socketvar.h>
84 #include <sys/random.h>
85 #if NECP
86 #include <net/necp.h>
87 #endif /* NECP */
88
89 #include <security/audit/audit.h>
90 #include <security/mac.h>
91 #include <bsm/audit_kevents.h>
92
93 #include <kern/kalloc.h>
94 #include <vm/vm_map.h>
95 #include <vm/vm_kern.h>
96 #include <vm/vm_pageout.h>
97
98 #include <mach/shared_region.h>
99 #include <vm/vm_shared_region.h>
100
101 #include <vm/vm_protos.h>
102
103 #include <sys/kern_memorystatus.h>
104 #include <sys/kern_memorystatus_freeze.h>
105 #include <sys/proc_internal.h>
106
107 #if CONFIG_MACF
108 #include <security/mac_framework.h>
109 #endif
110
111 #include <kern/bits.h>
112
113 #if CONFIG_CSR
114 #include <sys/csr.h>
115 #endif /* CONFIG_CSR */
116 #include <IOKit/IOBSD.h>
117
118 #if VM_MAP_DEBUG_APPLE_PROTECT
119 SYSCTL_INT(_vm, OID_AUTO, map_debug_apple_protect, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_map_debug_apple_protect, 0, "");
120 #endif /* VM_MAP_DEBUG_APPLE_PROTECT */
121
122 #if VM_MAP_DEBUG_FOURK
123 SYSCTL_INT(_vm, OID_AUTO, map_debug_fourk, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_map_debug_fourk, 0, "");
124 #endif /* VM_MAP_DEBUG_FOURK */
125
126 #if DEVELOPMENT || DEBUG
127
128 static int
129 sysctl_kmem_alloc_contig SYSCTL_HANDLER_ARGS
130 {
131 #pragma unused(arg1, arg2)
132 vm_offset_t kaddr;
133 kern_return_t kr;
134 int error = 0;
135 int size = 0;
136
137 error = sysctl_handle_int(oidp, &size, 0, req);
138 if (error || !req->newptr) {
139 return error;
140 }
141
142 kr = kmem_alloc_contig(kernel_map, &kaddr, (vm_size_t)size, 0, 0, 0, 0, VM_KERN_MEMORY_IOKIT);
143
144 if (kr == KERN_SUCCESS) {
145 kmem_free(kernel_map, kaddr, size);
146 }
147
148 return error;
149 }
150
151 SYSCTL_PROC(_vm, OID_AUTO, kmem_alloc_contig, CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED,
152 0, 0, &sysctl_kmem_alloc_contig, "I", "");
153
154 extern int vm_region_footprint;
155 SYSCTL_INT(_vm, OID_AUTO, region_footprint, CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_LOCKED, &vm_region_footprint, 0, "");
156
157 #endif /* DEVELOPMENT || DEBUG */
158
159 static int
160 sysctl_vm_self_region_footprint SYSCTL_HANDLER_ARGS
161 {
162 #pragma unused(arg1, arg2, oidp)
163 int error = 0;
164 int value;
165
166 value = task_self_region_footprint();
167 error = SYSCTL_OUT(req, &value, sizeof(int));
168 if (error) {
169 return error;
170 }
171
172 if (!req->newptr) {
173 return 0;
174 }
175
176 error = SYSCTL_IN(req, &value, sizeof(int));
177 if (error) {
178 return error;
179 }
180 task_self_region_footprint_set(value);
181 return 0;
182 }
183 SYSCTL_PROC(_vm, OID_AUTO, self_region_footprint, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_LOCKED | CTLFLAG_MASKED, 0, 0, &sysctl_vm_self_region_footprint, "I", "");
184
185 static int
186 sysctl_vm_self_region_page_size SYSCTL_HANDLER_ARGS
187 {
188 #pragma unused(arg1, arg2, oidp)
189 int error = 0;
190 int value;
191
192 value = (1 << thread_self_region_page_shift());
193 error = SYSCTL_OUT(req, &value, sizeof(int));
194 if (error) {
195 return error;
196 }
197
198 if (!req->newptr) {
199 return 0;
200 }
201
202 error = SYSCTL_IN(req, &value, sizeof(int));
203 if (error) {
204 return error;
205 }
206
207 if (value != 0 && value != 4096 && value != 16384) {
208 return EINVAL;
209 }
210
211 #if !__ARM_MIXED_PAGE_SIZE__
212 if (value != vm_map_page_size(current_map())) {
213 return EINVAL;
214 }
215 #endif /* !__ARM_MIXED_PAGE_SIZE__ */
216
217 thread_self_region_page_shift_set(bit_first(value));
218 return 0;
219 }
220 SYSCTL_PROC(_vm, OID_AUTO, self_region_page_size, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_LOCKED | CTLFLAG_MASKED, 0, 0, &sysctl_vm_self_region_page_size, "I", "");
221
222
223 #if DEVELOPMENT || DEBUG
224 extern int panic_on_unsigned_execute;
225 SYSCTL_INT(_vm, OID_AUTO, panic_on_unsigned_execute, CTLFLAG_RW | CTLFLAG_LOCKED, &panic_on_unsigned_execute, 0, "");
226 #endif /* DEVELOPMENT || DEBUG */
227
228 extern int cs_executable_create_upl;
229 extern int cs_executable_wire;
230 SYSCTL_INT(_vm, OID_AUTO, cs_executable_create_upl, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_create_upl, 0, "");
231 SYSCTL_INT(_vm, OID_AUTO, cs_executable_wire, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_wire, 0, "");
232
233 #if DEVELOPMENT || DEBUG
234 extern int radar_20146450;
235 SYSCTL_INT(_vm, OID_AUTO, radar_20146450, CTLFLAG_RW | CTLFLAG_LOCKED, &radar_20146450, 0, "");
236
237 extern int macho_printf;
238 SYSCTL_INT(_vm, OID_AUTO, macho_printf, CTLFLAG_RW | CTLFLAG_LOCKED, &macho_printf, 0, "");
239
240 extern int apple_protect_pager_data_request_debug;
241 SYSCTL_INT(_vm, OID_AUTO, apple_protect_pager_data_request_debug, CTLFLAG_RW | CTLFLAG_LOCKED, &apple_protect_pager_data_request_debug, 0, "");
242
243 #if __arm__ || __arm64__
244 /* These are meant to support the page table accounting unit test. */
245 extern unsigned int arm_hardware_page_size;
246 extern unsigned int arm_pt_desc_size;
247 extern unsigned int arm_pt_root_size;
248 extern unsigned int free_page_size_tt_count;
249 extern unsigned int free_two_page_size_tt_count;
250 extern unsigned int free_tt_count;
251 extern unsigned int inuse_user_tteroot_count;
252 extern unsigned int inuse_kernel_tteroot_count;
253 extern unsigned int inuse_user_ttepages_count;
254 extern unsigned int inuse_kernel_ttepages_count;
255 extern unsigned int inuse_user_ptepages_count;
256 extern unsigned int inuse_kernel_ptepages_count;
257 SYSCTL_UINT(_vm, OID_AUTO, native_hw_pagesize, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_hardware_page_size, 0, "");
258 SYSCTL_UINT(_vm, OID_AUTO, arm_pt_desc_size, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_pt_desc_size, 0, "");
259 SYSCTL_UINT(_vm, OID_AUTO, arm_pt_root_size, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_pt_root_size, 0, "");
260 SYSCTL_UINT(_vm, OID_AUTO, free_1page_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_page_size_tt_count, 0, "");
261 SYSCTL_UINT(_vm, OID_AUTO, free_2page_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_two_page_size_tt_count, 0, "");
262 SYSCTL_UINT(_vm, OID_AUTO, free_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_tt_count, 0, "");
263 SYSCTL_UINT(_vm, OID_AUTO, user_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_tteroot_count, 0, "");
264 SYSCTL_UINT(_vm, OID_AUTO, kernel_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_tteroot_count, 0, "");
265 SYSCTL_UINT(_vm, OID_AUTO, user_tte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_ttepages_count, 0, "");
266 SYSCTL_UINT(_vm, OID_AUTO, kernel_tte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_ttepages_count, 0, "");
267 SYSCTL_UINT(_vm, OID_AUTO, user_pte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_ptepages_count, 0, "");
268 SYSCTL_UINT(_vm, OID_AUTO, kernel_pte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_ptepages_count, 0, "");
269 #if DEVELOPMENT || DEBUG
270 extern unsigned long pmap_asid_flushes;
271 SYSCTL_ULONG(_vm, OID_AUTO, pmap_asid_flushes, CTLFLAG_RD | CTLFLAG_LOCKED, &pmap_asid_flushes, "");
272 extern unsigned long pmap_asid_hits;
273 SYSCTL_ULONG(_vm, OID_AUTO, pmap_asid_hits, CTLFLAG_RD | CTLFLAG_LOCKED, &pmap_asid_hits, "");
274 extern unsigned long pmap_asid_misses;
275 SYSCTL_ULONG(_vm, OID_AUTO, pmap_asid_misses, CTLFLAG_RD | CTLFLAG_LOCKED, &pmap_asid_misses, "");
276 #endif
277 #endif /* __arm__ || __arm64__ */
278
279 #if __arm64__
280 extern int fourk_pager_data_request_debug;
281 SYSCTL_INT(_vm, OID_AUTO, fourk_pager_data_request_debug, CTLFLAG_RW | CTLFLAG_LOCKED, &fourk_pager_data_request_debug, 0, "");
282 #endif /* __arm64__ */
283 #endif /* DEVELOPMENT || DEBUG */
284
285 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_compressor, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_compressor, 0, "");
286 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_compressor_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_compressor_pages, 0, "");
287 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_terminate, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_terminate, 0, "");
288 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_terminate_failure, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_terminate_failure, 0, "");
289 SYSCTL_INT(_vm, OID_AUTO, vm_should_cow_but_wired, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.should_cow_but_wired, 0, "");
290 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_extra_cow, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_extra_cow, 0, "");
291 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_extra_cow_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_extra_cow_pages, 0, "");
292 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_lookup_failure_write, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_lookup_failure_write, 0, "");
293 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_lookup_failure_copy, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_lookup_failure_copy, 0, "");
294 #if VM_SCAN_FOR_SHADOW_CHAIN
295 static int vm_shadow_max_enabled = 0; /* Disabled by default */
296 extern int proc_shadow_max(void);
297 static int
298 vm_shadow_max SYSCTL_HANDLER_ARGS
299 {
300 #pragma unused(arg1, arg2, oidp)
301 int value = 0;
302
303 if (vm_shadow_max_enabled) {
304 value = proc_shadow_max();
305 }
306
307 return SYSCTL_OUT(req, &value, sizeof(value));
308 }
309 SYSCTL_PROC(_vm, OID_AUTO, vm_shadow_max, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
310 0, 0, &vm_shadow_max, "I", "");
311
312 SYSCTL_INT(_vm, OID_AUTO, vm_shadow_max_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_shadow_max_enabled, 0, "");
313
314 #endif /* VM_SCAN_FOR_SHADOW_CHAIN */
315
316 SYSCTL_INT(_vm, OID_AUTO, vm_debug_events, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_debug_events, 0, "");
317
318 __attribute__((noinline)) int __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(
319 mach_port_t task_access_port, int32_t calling_pid, uint32_t calling_gid, int32_t target_pid);
320 /*
321 * Sysctl's related to data/stack execution. See osfmk/vm/vm_map.c
322 */
323
324 #if DEVELOPMENT || DEBUG
325 extern int allow_stack_exec, allow_data_exec;
326
327 SYSCTL_INT(_vm, OID_AUTO, allow_stack_exec, CTLFLAG_RW | CTLFLAG_LOCKED, &allow_stack_exec, 0, "");
328 SYSCTL_INT(_vm, OID_AUTO, allow_data_exec, CTLFLAG_RW | CTLFLAG_LOCKED, &allow_data_exec, 0, "");
329
330 #endif /* DEVELOPMENT || DEBUG */
331
332 static const char *prot_values[] = {
333 "none",
334 "read-only",
335 "write-only",
336 "read-write",
337 "execute-only",
338 "read-execute",
339 "write-execute",
340 "read-write-execute"
341 };
342
343 void
344 log_stack_execution_failure(addr64_t vaddr, vm_prot_t prot)
345 {
346 printf("Data/Stack execution not permitted: %s[pid %d] at virtual address 0x%qx, protections were %s\n",
347 current_proc()->p_comm, current_proc()->p_pid, vaddr, prot_values[prot & VM_PROT_ALL]);
348 }
349
350 /*
351 * shared_region_unnest_logging: level of logging of unnesting events
352 * 0 - no logging
353 * 1 - throttled logging of unexpected unnesting events (default)
354 * 2 - unthrottled logging of unexpected unnesting events
355 * 3+ - unthrottled logging of all unnesting events
356 */
357 int shared_region_unnest_logging = 1;
358
359 SYSCTL_INT(_vm, OID_AUTO, shared_region_unnest_logging, CTLFLAG_RW | CTLFLAG_LOCKED,
360 &shared_region_unnest_logging, 0, "");
361
362 int vm_shared_region_unnest_log_interval = 10;
363 int shared_region_unnest_log_count_threshold = 5;
364
365 /*
366 * Shared cache path enforcement.
367 */
368
369 #if XNU_TARGET_OS_OSX
370
371 #if defined (__x86_64__)
372 static int scdir_enforce = 1;
373 #else /* defined (__x86_64__) */
374 static int scdir_enforce = 0; /* AOT caches live elsewhere */
375 #endif /* defined (__x86_64__) */
376
377 static char scdir_path[] = "/System/Library/dyld/";
378
379 #else /* XNU_TARGET_OS_OSX */
380
381 static int scdir_enforce = 0;
382 static char scdir_path[] = "/System/Library/Caches/com.apple.dyld/";
383
384 #endif /* XNU_TARGET_OS_OSX */
385
386 #ifndef SECURE_KERNEL
387 static int sysctl_scdir_enforce SYSCTL_HANDLER_ARGS
388 {
389 #if CONFIG_CSR
390 if (csr_check(CSR_ALLOW_UNRESTRICTED_FS) != 0) {
391 printf("Failed attempt to set vm.enforce_shared_cache_dir sysctl\n");
392 return EPERM;
393 }
394 #endif /* CONFIG_CSR */
395 return sysctl_handle_int(oidp, arg1, arg2, req);
396 }
397
398 SYSCTL_PROC(_vm, OID_AUTO, enforce_shared_cache_dir, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &scdir_enforce, 0, sysctl_scdir_enforce, "I", "");
399 #endif
400
401 /* These log rate throttling state variables aren't thread safe, but
402 * are sufficient unto the task.
403 */
404 static int64_t last_unnest_log_time = 0;
405 static int shared_region_unnest_log_count = 0;
406
407 void
408 log_unnest_badness(
409 vm_map_t m,
410 vm_map_offset_t s,
411 vm_map_offset_t e,
412 boolean_t is_nested_map,
413 vm_map_offset_t lowest_unnestable_addr)
414 {
415 struct timeval tv;
416
417 if (shared_region_unnest_logging == 0) {
418 return;
419 }
420
421 if (shared_region_unnest_logging <= 2 &&
422 is_nested_map &&
423 s >= lowest_unnestable_addr) {
424 /*
425 * Unnesting of writable map entries is fine.
426 */
427 return;
428 }
429
430 if (shared_region_unnest_logging <= 1) {
431 microtime(&tv);
432 if ((tv.tv_sec - last_unnest_log_time) <
433 vm_shared_region_unnest_log_interval) {
434 if (shared_region_unnest_log_count++ >
435 shared_region_unnest_log_count_threshold) {
436 return;
437 }
438 } else {
439 last_unnest_log_time = tv.tv_sec;
440 shared_region_unnest_log_count = 0;
441 }
442 }
443
444 DTRACE_VM4(log_unnest_badness,
445 vm_map_t, m,
446 vm_map_offset_t, s,
447 vm_map_offset_t, e,
448 vm_map_offset_t, lowest_unnestable_addr);
449 printf("%s[%d] triggered unnest of range 0x%qx->0x%qx of DYLD shared region in VM map %p. While not abnormal for debuggers, this increases system memory footprint until the target exits.\n", current_proc()->p_comm, current_proc()->p_pid, (uint64_t)s, (uint64_t)e, (void *) VM_KERNEL_ADDRPERM(m));
450 }
451
452 int
453 useracc(
454 user_addr_t addr,
455 user_size_t len,
456 int prot)
457 {
458 vm_map_t map;
459
460 map = current_map();
461 return vm_map_check_protection(
462 map,
463 vm_map_trunc_page(addr,
464 vm_map_page_mask(map)),
465 vm_map_round_page(addr + len,
466 vm_map_page_mask(map)),
467 prot == B_READ ? VM_PROT_READ : VM_PROT_WRITE);
468 }
469
470 int
471 vslock(
472 user_addr_t addr,
473 user_size_t len)
474 {
475 kern_return_t kret;
476 vm_map_t map;
477
478 map = current_map();
479 kret = vm_map_wire_kernel(map,
480 vm_map_trunc_page(addr,
481 vm_map_page_mask(map)),
482 vm_map_round_page(addr + len,
483 vm_map_page_mask(map)),
484 VM_PROT_READ | VM_PROT_WRITE, VM_KERN_MEMORY_BSD,
485 FALSE);
486
487 switch (kret) {
488 case KERN_SUCCESS:
489 return 0;
490 case KERN_INVALID_ADDRESS:
491 case KERN_NO_SPACE:
492 return ENOMEM;
493 case KERN_PROTECTION_FAILURE:
494 return EACCES;
495 default:
496 return EINVAL;
497 }
498 }
499
500 int
501 vsunlock(
502 user_addr_t addr,
503 user_size_t len,
504 __unused int dirtied)
505 {
506 #if FIXME /* [ */
507 pmap_t pmap;
508 vm_page_t pg;
509 vm_map_offset_t vaddr;
510 ppnum_t paddr;
511 #endif /* FIXME ] */
512 kern_return_t kret;
513 vm_map_t map;
514
515 map = current_map();
516
517 #if FIXME /* [ */
518 if (dirtied) {
519 pmap = get_task_pmap(current_task());
520 for (vaddr = vm_map_trunc_page(addr, PAGE_MASK);
521 vaddr < vm_map_round_page(addr + len, PAGE_MASK);
522 vaddr += PAGE_SIZE) {
523 paddr = pmap_find_phys(pmap, vaddr);
524 pg = PHYS_TO_VM_PAGE(paddr);
525 vm_page_set_modified(pg);
526 }
527 }
528 #endif /* FIXME ] */
529 #ifdef lint
530 dirtied++;
531 #endif /* lint */
532 kret = vm_map_unwire(map,
533 vm_map_trunc_page(addr,
534 vm_map_page_mask(map)),
535 vm_map_round_page(addr + len,
536 vm_map_page_mask(map)),
537 FALSE);
538 switch (kret) {
539 case KERN_SUCCESS:
540 return 0;
541 case KERN_INVALID_ADDRESS:
542 case KERN_NO_SPACE:
543 return ENOMEM;
544 case KERN_PROTECTION_FAILURE:
545 return EACCES;
546 default:
547 return EINVAL;
548 }
549 }
550
551 int
552 subyte(
553 user_addr_t addr,
554 int byte)
555 {
556 char character;
557
558 character = (char)byte;
559 return copyout((void *)&(character), addr, sizeof(char)) == 0 ? 0 : -1;
560 }
561
562 int
563 suibyte(
564 user_addr_t addr,
565 int byte)
566 {
567 char character;
568
569 character = (char)byte;
570 return copyout((void *)&(character), addr, sizeof(char)) == 0 ? 0 : -1;
571 }
572
573 int
574 fubyte(user_addr_t addr)
575 {
576 unsigned char byte;
577
578 if (copyin(addr, (void *) &byte, sizeof(char))) {
579 return -1;
580 }
581 return byte;
582 }
583
584 int
585 fuibyte(user_addr_t addr)
586 {
587 unsigned char byte;
588
589 if (copyin(addr, (void *) &(byte), sizeof(char))) {
590 return -1;
591 }
592 return byte;
593 }
594
595 int
596 suword(
597 user_addr_t addr,
598 long word)
599 {
600 return copyout((void *) &word, addr, sizeof(int)) == 0 ? 0 : -1;
601 }
602
603 long
604 fuword(user_addr_t addr)
605 {
606 long word = 0;
607
608 if (copyin(addr, (void *) &word, sizeof(int))) {
609 return -1;
610 }
611 return word;
612 }
613
614 /* suiword and fuiword are the same as suword and fuword, respectively */
615
616 int
617 suiword(
618 user_addr_t addr,
619 long word)
620 {
621 return copyout((void *) &word, addr, sizeof(int)) == 0 ? 0 : -1;
622 }
623
624 long
625 fuiword(user_addr_t addr)
626 {
627 long word = 0;
628
629 if (copyin(addr, (void *) &word, sizeof(int))) {
630 return -1;
631 }
632 return word;
633 }
634
635 /*
636 * With a 32-bit kernel and mixed 32/64-bit user tasks, this interface allows the
637 * fetching and setting of process-sized size_t and pointer values.
638 */
639 int
640 sulong(user_addr_t addr, int64_t word)
641 {
642 if (IS_64BIT_PROCESS(current_proc())) {
643 return copyout((void *)&word, addr, sizeof(word)) == 0 ? 0 : -1;
644 } else {
645 return suiword(addr, (long)word);
646 }
647 }
648
649 int64_t
650 fulong(user_addr_t addr)
651 {
652 int64_t longword;
653
654 if (IS_64BIT_PROCESS(current_proc())) {
655 if (copyin(addr, (void *)&longword, sizeof(longword)) != 0) {
656 return -1;
657 }
658 return longword;
659 } else {
660 return (int64_t)fuiword(addr);
661 }
662 }
663
664 int
665 suulong(user_addr_t addr, uint64_t uword)
666 {
667 if (IS_64BIT_PROCESS(current_proc())) {
668 return copyout((void *)&uword, addr, sizeof(uword)) == 0 ? 0 : -1;
669 } else {
670 return suiword(addr, (uint32_t)uword);
671 }
672 }
673
674 uint64_t
675 fuulong(user_addr_t addr)
676 {
677 uint64_t ulongword;
678
679 if (IS_64BIT_PROCESS(current_proc())) {
680 if (copyin(addr, (void *)&ulongword, sizeof(ulongword)) != 0) {
681 return -1ULL;
682 }
683 return ulongword;
684 } else {
685 return (uint64_t)fuiword(addr);
686 }
687 }
688
689 int
690 swapon(__unused proc_t procp, __unused struct swapon_args *uap, __unused int *retval)
691 {
692 return ENOTSUP;
693 }
694
695 /*
696 * pid_for_task
697 *
698 * Find the BSD process ID for the Mach task associated with the given Mach port
699 * name
700 *
701 * Parameters: args User argument descriptor (see below)
702 *
703 * Indirect parameters: args->t Mach port name
704 * args->pid Process ID (returned value; see below)
705 *
706 * Returns: KERL_SUCCESS Success
707 * KERN_FAILURE Not success
708 *
709 * Implicit returns: args->pid Process ID
710 *
711 */
712 kern_return_t
713 pid_for_task(
714 struct pid_for_task_args *args)
715 {
716 mach_port_name_t t = args->t;
717 user_addr_t pid_addr = args->pid;
718 proc_t p;
719 task_t t1;
720 int pid = -1;
721 kern_return_t err = KERN_SUCCESS;
722
723 AUDIT_MACH_SYSCALL_ENTER(AUE_PIDFORTASK);
724 AUDIT_ARG(mach_port1, t);
725
726 t1 = port_name_to_task_name(t);
727
728 if (t1 == TASK_NULL) {
729 err = KERN_FAILURE;
730 goto pftout;
731 } else {
732 p = get_bsdtask_info(t1);
733 if (p) {
734 pid = proc_pid(p);
735 err = KERN_SUCCESS;
736 } else if (is_corpsetask(t1)) {
737 pid = task_pid(t1);
738 err = KERN_SUCCESS;
739 } else {
740 err = KERN_FAILURE;
741 }
742 }
743 task_deallocate(t1);
744 pftout:
745 AUDIT_ARG(pid, pid);
746 (void) copyout((char *) &pid, pid_addr, sizeof(int));
747 AUDIT_MACH_SYSCALL_EXIT(err);
748 return err;
749 }
750
751 /*
752 *
753 * tfp_policy = KERN_TFP_POLICY_DENY; Deny Mode: None allowed except for self
754 * tfp_policy = KERN_TFP_POLICY_DEFAULT; default mode: all posix checks and upcall via task port for authentication
755 *
756 */
757 static int tfp_policy = KERN_TFP_POLICY_DEFAULT;
758
759 /*
760 * Routine: task_for_pid_posix_check
761 * Purpose:
762 * Verify that the current process should be allowed to
763 * get the target process's task port. This is only
764 * permitted if:
765 * - The current process is root
766 * OR all of the following are true:
767 * - The target process's real, effective, and saved uids
768 * are the same as the current proc's euid,
769 * - The target process's group set is a subset of the
770 * calling process's group set, and
771 * - The target process hasn't switched credentials.
772 *
773 * Returns: TRUE: permitted
774 * FALSE: denied
775 */
776 static int
777 task_for_pid_posix_check(proc_t target)
778 {
779 kauth_cred_t targetcred, mycred;
780 uid_t myuid;
781 int allowed;
782
783 /* No task_for_pid on bad targets */
784 if (target->p_stat == SZOMB) {
785 return FALSE;
786 }
787
788 mycred = kauth_cred_get();
789 myuid = kauth_cred_getuid(mycred);
790
791 /* If we're running as root, the check passes */
792 if (kauth_cred_issuser(mycred)) {
793 return TRUE;
794 }
795
796 /* We're allowed to get our own task port */
797 if (target == current_proc()) {
798 return TRUE;
799 }
800
801 /*
802 * Under DENY, only root can get another proc's task port,
803 * so no more checks are needed.
804 */
805 if (tfp_policy == KERN_TFP_POLICY_DENY) {
806 return FALSE;
807 }
808
809 targetcred = kauth_cred_proc_ref(target);
810 allowed = TRUE;
811
812 /* Do target's ruid, euid, and saved uid match my euid? */
813 if ((kauth_cred_getuid(targetcred) != myuid) ||
814 (kauth_cred_getruid(targetcred) != myuid) ||
815 (kauth_cred_getsvuid(targetcred) != myuid)) {
816 allowed = FALSE;
817 goto out;
818 }
819
820 /* Are target's groups a subset of my groups? */
821 if (kauth_cred_gid_subset(targetcred, mycred, &allowed) ||
822 allowed == 0) {
823 allowed = FALSE;
824 goto out;
825 }
826
827 /* Has target switched credentials? */
828 if (target->p_flag & P_SUGID) {
829 allowed = FALSE;
830 goto out;
831 }
832
833 out:
834 kauth_cred_unref(&targetcred);
835 return allowed;
836 }
837
838 /*
839 * __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__
840 *
841 * Description: Waits for the user space daemon to respond to the request
842 * we made. Function declared non inline to be visible in
843 * stackshots and spindumps as well as debugging.
844 */
845 __attribute__((noinline)) int
846 __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(
847 mach_port_t task_access_port, int32_t calling_pid, uint32_t calling_gid, int32_t target_pid)
848 {
849 return check_task_access(task_access_port, calling_pid, calling_gid, target_pid);
850 }
851
852 /*
853 * Routine: task_for_pid
854 * Purpose:
855 * Get the task port for another "process", named by its
856 * process ID on the same host as "target_task".
857 *
858 * Only permitted to privileged processes, or processes
859 * with the same user ID.
860 *
861 * Note: if pid == 0, an error is return no matter who is calling.
862 *
863 * XXX This should be a BSD system call, not a Mach trap!!!
864 */
865 kern_return_t
866 task_for_pid(
867 struct task_for_pid_args *args)
868 {
869 mach_port_name_t target_tport = args->target_tport;
870 int pid = args->pid;
871 user_addr_t task_addr = args->t;
872 proc_t p = PROC_NULL;
873 task_t t1 = TASK_NULL;
874 task_t task = TASK_NULL;
875 mach_port_name_t tret = MACH_PORT_NULL;
876 ipc_port_t tfpport = MACH_PORT_NULL;
877 void * sright = NULL;
878 int error = 0;
879 boolean_t is_current_proc = FALSE;
880 struct proc_ident pident = {0};
881
882 AUDIT_MACH_SYSCALL_ENTER(AUE_TASKFORPID);
883 AUDIT_ARG(pid, pid);
884 AUDIT_ARG(mach_port1, target_tport);
885
886 /* Always check if pid == 0 */
887 if (pid == 0) {
888 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
889 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
890 return KERN_FAILURE;
891 }
892
893 t1 = port_name_to_task(target_tport);
894 if (t1 == TASK_NULL) {
895 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
896 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
897 return KERN_FAILURE;
898 }
899
900
901 p = proc_find(pid);
902 if (p == PROC_NULL) {
903 error = KERN_FAILURE;
904 goto tfpout;
905 }
906 pident = proc_ident(p);
907 is_current_proc = (p == current_proc());
908
909 #if CONFIG_AUDIT
910 AUDIT_ARG(process, p);
911 #endif
912
913 if (!(task_for_pid_posix_check(p))) {
914 error = KERN_FAILURE;
915 goto tfpout;
916 }
917
918 if (p->task == TASK_NULL) {
919 error = KERN_SUCCESS;
920 goto tfpout;
921 }
922
923 /*
924 * Grab a task reference and drop the proc reference as the proc ref
925 * shouldn't be held accross upcalls.
926 */
927 task = p->task;
928 task_reference(task);
929
930 proc_rele(p);
931 p = PROC_NULL;
932
933 #if CONFIG_MACF
934 error = mac_proc_check_get_task(kauth_cred_get(), &pident);
935 if (error) {
936 error = KERN_FAILURE;
937 goto tfpout;
938 }
939 #endif
940
941 /* If we aren't root and target's task access port is set... */
942 if (!kauth_cred_issuser(kauth_cred_get()) &&
943 !is_current_proc &&
944 (task_get_task_access_port(task, &tfpport) == 0) &&
945 (tfpport != IPC_PORT_NULL)) {
946 if (tfpport == IPC_PORT_DEAD) {
947 error = KERN_PROTECTION_FAILURE;
948 goto tfpout;
949 }
950
951 /* Call up to the task access server */
952 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
953
954 if (error != MACH_MSG_SUCCESS) {
955 if (error == MACH_RCV_INTERRUPTED) {
956 error = KERN_ABORTED;
957 } else {
958 error = KERN_FAILURE;
959 }
960 goto tfpout;
961 }
962 }
963
964 /* Grant task port access */
965 extmod_statistics_incr_task_for_pid(task);
966 sright = (void *) convert_task_to_port(task);
967
968 /* Check if the task has been corpsified */
969 if (is_corpsetask(task)) {
970 /* task ref consumed by convert_task_to_port */
971 task = TASK_NULL;
972 ipc_port_release_send(sright);
973 error = KERN_FAILURE;
974 goto tfpout;
975 }
976
977 /* task ref consumed by convert_task_to_port */
978 task = TASK_NULL;
979 tret = ipc_port_copyout_send(
980 sright,
981 get_task_ipcspace(current_task()));
982
983 error = KERN_SUCCESS;
984
985 tfpout:
986 task_deallocate(t1);
987 AUDIT_ARG(mach_port2, tret);
988 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
989
990 if (tfpport != IPC_PORT_NULL) {
991 ipc_port_release_send(tfpport);
992 }
993 if (task != TASK_NULL) {
994 task_deallocate(task);
995 }
996 if (p != PROC_NULL) {
997 proc_rele(p);
998 }
999 AUDIT_MACH_SYSCALL_EXIT(error);
1000 return error;
1001 }
1002
1003 /*
1004 * Routine: task_name_for_pid
1005 * Purpose:
1006 * Get the task name port for another "process", named by its
1007 * process ID on the same host as "target_task".
1008 *
1009 * Only permitted to privileged processes, or processes
1010 * with the same user ID.
1011 *
1012 * XXX This should be a BSD system call, not a Mach trap!!!
1013 */
1014
1015 kern_return_t
1016 task_name_for_pid(
1017 struct task_name_for_pid_args *args)
1018 {
1019 mach_port_name_t target_tport = args->target_tport;
1020 int pid = args->pid;
1021 user_addr_t task_addr = args->t;
1022 proc_t p = PROC_NULL;
1023 task_t t1;
1024 mach_port_name_t tret;
1025 void * sright;
1026 int error = 0, refheld = 0;
1027 kauth_cred_t target_cred;
1028
1029 AUDIT_MACH_SYSCALL_ENTER(AUE_TASKNAMEFORPID);
1030 AUDIT_ARG(pid, pid);
1031 AUDIT_ARG(mach_port1, target_tport);
1032
1033 t1 = port_name_to_task(target_tport);
1034 if (t1 == TASK_NULL) {
1035 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
1036 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
1037 return KERN_FAILURE;
1038 }
1039
1040 p = proc_find(pid);
1041 if (p != PROC_NULL) {
1042 AUDIT_ARG(process, p);
1043 target_cred = kauth_cred_proc_ref(p);
1044 refheld = 1;
1045
1046 if ((p->p_stat != SZOMB)
1047 && ((current_proc() == p)
1048 || kauth_cred_issuser(kauth_cred_get())
1049 || ((kauth_cred_getuid(target_cred) == kauth_cred_getuid(kauth_cred_get())) &&
1050 ((kauth_cred_getruid(target_cred) == kauth_getruid()))))) {
1051 if (p->task != TASK_NULL) {
1052 struct proc_ident pident = proc_ident(p);
1053
1054 task_t task = p->task;
1055
1056 task_reference(p->task);
1057 proc_rele(p);
1058 p = PROC_NULL;
1059 #if CONFIG_MACF
1060 error = mac_proc_check_get_task_name(kauth_cred_get(), &pident);
1061 if (error) {
1062 task_deallocate(task);
1063 goto noperm;
1064 }
1065 #endif
1066 sright = (void *)convert_task_name_to_port(task);
1067 task = NULL;
1068 tret = ipc_port_copyout_send(sright,
1069 get_task_ipcspace(current_task()));
1070 } else {
1071 tret = MACH_PORT_NULL;
1072 }
1073
1074 AUDIT_ARG(mach_port2, tret);
1075 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
1076 task_deallocate(t1);
1077 error = KERN_SUCCESS;
1078 goto tnfpout;
1079 }
1080 }
1081
1082 #if CONFIG_MACF
1083 noperm:
1084 #endif
1085 task_deallocate(t1);
1086 tret = MACH_PORT_NULL;
1087 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1088 error = KERN_FAILURE;
1089 tnfpout:
1090 if (refheld != 0) {
1091 kauth_cred_unref(&target_cred);
1092 }
1093 if (p != PROC_NULL) {
1094 proc_rele(p);
1095 }
1096 AUDIT_MACH_SYSCALL_EXIT(error);
1097 return error;
1098 }
1099
1100 /*
1101 * Routine: task_inspect_for_pid
1102 * Purpose:
1103 * Get the task inspect port for another "process", named by its
1104 * process ID on the same host as "target_task".
1105 */
1106 int
1107 task_inspect_for_pid(struct proc *p __unused, struct task_inspect_for_pid_args *args, int *ret)
1108 {
1109 mach_port_name_t target_tport = args->target_tport;
1110 int pid = args->pid;
1111 user_addr_t task_addr = args->t;
1112
1113 proc_t proc = PROC_NULL;
1114 task_t t1 = TASK_NULL;
1115 task_inspect_t task_insp = TASK_INSPECT_NULL;
1116 mach_port_name_t tret = MACH_PORT_NULL;
1117 ipc_port_t tfpport = MACH_PORT_NULL;
1118 int error = 0;
1119 void *sright = NULL;
1120 boolean_t is_current_proc = FALSE;
1121 struct proc_ident pident = {0};
1122
1123 /* Disallow inspect port for kernel_task */
1124 if (pid == 0) {
1125 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
1126 return EPERM;
1127 }
1128
1129 t1 = port_name_to_task(target_tport);
1130 if (t1 == TASK_NULL) {
1131 (void) copyout((char *) &t1, task_addr, sizeof(mach_port_name_t));
1132 return EINVAL;
1133 }
1134
1135 proc = proc_find(pid);
1136 if (proc == PROC_NULL) {
1137 error = ESRCH;
1138 goto tifpout;
1139 }
1140 pident = proc_ident(proc);
1141 is_current_proc = (proc == current_proc());
1142
1143 if (!(task_for_pid_posix_check(proc))) {
1144 error = EPERM;
1145 goto tifpout;
1146 }
1147
1148 task_insp = proc->task;
1149 if (task_insp == TASK_INSPECT_NULL) {
1150 goto tifpout;
1151 }
1152
1153 /*
1154 * Grab a task reference and drop the proc reference before making any upcalls.
1155 */
1156 task_reference(task_insp);
1157
1158 proc_rele(proc);
1159 proc = PROC_NULL;
1160
1161 /*
1162 * For now, it performs the same set of permission checks as task_for_pid. This
1163 * will be addressed in rdar://problem/53478660
1164 */
1165 #if CONFIG_MACF
1166 error = mac_proc_check_get_task(kauth_cred_get(), &pident);
1167 if (error) {
1168 error = EPERM;
1169 goto tifpout;
1170 }
1171 #endif
1172
1173 /* If we aren't root and target's task access port is set... */
1174 if (!kauth_cred_issuser(kauth_cred_get()) &&
1175 !is_current_proc &&
1176 (task_get_task_access_port(task_insp, &tfpport) == 0) &&
1177 (tfpport != IPC_PORT_NULL)) {
1178 if (tfpport == IPC_PORT_DEAD) {
1179 error = EACCES;
1180 goto tifpout;
1181 }
1182
1183
1184 /* Call up to the task access server */
1185 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
1186
1187 if (error != MACH_MSG_SUCCESS) {
1188 if (error == MACH_RCV_INTERRUPTED) {
1189 error = EINTR;
1190 } else {
1191 error = EPERM;
1192 }
1193 goto tifpout;
1194 }
1195 }
1196
1197 /* Check if the task has been corpsified */
1198 if (is_corpsetask(task_insp)) {
1199 error = EACCES;
1200 goto tifpout;
1201 }
1202
1203 /* could be IP_NULL, consumes a ref */
1204 sright = (void*) convert_task_inspect_to_port(task_insp);
1205 task_insp = TASK_INSPECT_NULL;
1206 tret = ipc_port_copyout_send(sright, get_task_ipcspace(current_task()));
1207
1208 tifpout:
1209 task_deallocate(t1);
1210 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1211 if (proc != PROC_NULL) {
1212 proc_rele(proc);
1213 }
1214 if (tfpport != IPC_PORT_NULL) {
1215 ipc_port_release_send(tfpport);
1216 }
1217 if (task_insp != TASK_INSPECT_NULL) {
1218 task_deallocate(task_insp);
1219 }
1220
1221 *ret = error;
1222 return error;
1223 }
1224
1225 /*
1226 * Routine: task_read_for_pid
1227 * Purpose:
1228 * Get the task read port for another "process", named by its
1229 * process ID on the same host as "target_task".
1230 */
1231 int
1232 task_read_for_pid(struct proc *p __unused, struct task_read_for_pid_args *args, int *ret)
1233 {
1234 mach_port_name_t target_tport = args->target_tport;
1235 int pid = args->pid;
1236 user_addr_t task_addr = args->t;
1237
1238 proc_t proc = PROC_NULL;
1239 task_t t1 = TASK_NULL;
1240 task_read_t task_read = TASK_READ_NULL;
1241 mach_port_name_t tret = MACH_PORT_NULL;
1242 ipc_port_t tfpport = MACH_PORT_NULL;
1243 int error = 0;
1244 void *sright = NULL;
1245 boolean_t is_current_proc = FALSE;
1246 struct proc_ident pident = {0};
1247
1248 /* Disallow read port for kernel_task */
1249 if (pid == 0) {
1250 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
1251 return EPERM;
1252 }
1253
1254 t1 = port_name_to_task(target_tport);
1255 if (t1 == TASK_NULL) {
1256 (void) copyout((char *) &t1, task_addr, sizeof(mach_port_name_t));
1257 return EINVAL;
1258 }
1259
1260 proc = proc_find(pid);
1261 if (proc == PROC_NULL) {
1262 error = ESRCH;
1263 goto trfpout;
1264 }
1265 pident = proc_ident(proc);
1266 is_current_proc = (proc == current_proc());
1267
1268 if (!(task_for_pid_posix_check(proc))) {
1269 error = EPERM;
1270 goto trfpout;
1271 }
1272
1273 task_read = proc->task;
1274 if (task_read == TASK_INSPECT_NULL) {
1275 goto trfpout;
1276 }
1277
1278 /*
1279 * Grab a task reference and drop the proc reference before making any upcalls.
1280 */
1281 task_reference(task_read);
1282
1283 proc_rele(proc);
1284 proc = PROC_NULL;
1285
1286 /*
1287 * For now, it performs the same set of permission checks as task_for_pid. This
1288 * will be addressed in rdar://problem/53478660
1289 */
1290 #if CONFIG_MACF
1291 error = mac_proc_check_get_task(kauth_cred_get(), &pident);
1292 if (error) {
1293 error = EPERM;
1294 goto trfpout;
1295 }
1296 #endif
1297
1298 /* If we aren't root and target's task access port is set... */
1299 if (!kauth_cred_issuser(kauth_cred_get()) &&
1300 !is_current_proc &&
1301 (task_get_task_access_port(task_read, &tfpport) == 0) &&
1302 (tfpport != IPC_PORT_NULL)) {
1303 if (tfpport == IPC_PORT_DEAD) {
1304 error = EACCES;
1305 goto trfpout;
1306 }
1307
1308
1309 /* Call up to the task access server */
1310 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
1311
1312 if (error != MACH_MSG_SUCCESS) {
1313 if (error == MACH_RCV_INTERRUPTED) {
1314 error = EINTR;
1315 } else {
1316 error = EPERM;
1317 }
1318 goto trfpout;
1319 }
1320 }
1321
1322 /* Check if the task has been corpsified */
1323 if (is_corpsetask(task_read)) {
1324 error = EACCES;
1325 goto trfpout;
1326 }
1327
1328 /* could be IP_NULL, consumes a ref */
1329 sright = (void*) convert_task_read_to_port(task_read);
1330 task_read = TASK_READ_NULL;
1331 tret = ipc_port_copyout_send(sright, get_task_ipcspace(current_task()));
1332
1333 trfpout:
1334 task_deallocate(t1);
1335 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1336 if (proc != PROC_NULL) {
1337 proc_rele(proc);
1338 }
1339 if (tfpport != IPC_PORT_NULL) {
1340 ipc_port_release_send(tfpport);
1341 }
1342 if (task_read != TASK_READ_NULL) {
1343 task_deallocate(task_read);
1344 }
1345
1346 *ret = error;
1347 return error;
1348 }
1349
1350 kern_return_t
1351 pid_suspend(struct proc *p __unused, struct pid_suspend_args *args, int *ret)
1352 {
1353 task_t target = NULL;
1354 proc_t targetproc = PROC_NULL;
1355 int pid = args->pid;
1356 int error = 0;
1357 mach_port_t tfpport = MACH_PORT_NULL;
1358
1359 if (pid == 0) {
1360 error = EPERM;
1361 goto out;
1362 }
1363
1364 targetproc = proc_find(pid);
1365 if (targetproc == PROC_NULL) {
1366 error = ESRCH;
1367 goto out;
1368 }
1369
1370 if (!task_for_pid_posix_check(targetproc) &&
1371 !IOTaskHasEntitlement(current_task(), PROCESS_RESUME_SUSPEND_ENTITLEMENT)) {
1372 error = EPERM;
1373 goto out;
1374 }
1375
1376 #if CONFIG_MACF
1377 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_SUSPEND);
1378 if (error) {
1379 error = EPERM;
1380 goto out;
1381 }
1382 #endif
1383
1384 target = targetproc->task;
1385 #ifndef CONFIG_EMBEDDED
1386 if (target != TASK_NULL) {
1387 /* If we aren't root and target's task access port is set... */
1388 if (!kauth_cred_issuser(kauth_cred_get()) &&
1389 targetproc != current_proc() &&
1390 (task_get_task_access_port(target, &tfpport) == 0) &&
1391 (tfpport != IPC_PORT_NULL)) {
1392 if (tfpport == IPC_PORT_DEAD) {
1393 error = EACCES;
1394 goto out;
1395 }
1396
1397 /* Call up to the task access server */
1398 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
1399
1400 if (error != MACH_MSG_SUCCESS) {
1401 if (error == MACH_RCV_INTERRUPTED) {
1402 error = EINTR;
1403 } else {
1404 error = EPERM;
1405 }
1406 goto out;
1407 }
1408 }
1409 }
1410 #endif
1411
1412 task_reference(target);
1413 error = task_pidsuspend(target);
1414 if (error) {
1415 if (error == KERN_INVALID_ARGUMENT) {
1416 error = EINVAL;
1417 } else {
1418 error = EPERM;
1419 }
1420 }
1421 #if CONFIG_MEMORYSTATUS
1422 else {
1423 memorystatus_on_suspend(targetproc);
1424 }
1425 #endif
1426
1427 task_deallocate(target);
1428
1429 out:
1430 if (tfpport != IPC_PORT_NULL) {
1431 ipc_port_release_send(tfpport);
1432 }
1433
1434 if (targetproc != PROC_NULL) {
1435 proc_rele(targetproc);
1436 }
1437 *ret = error;
1438 return error;
1439 }
1440
1441 kern_return_t
1442 debug_control_port_for_pid(struct debug_control_port_for_pid_args *args)
1443 {
1444 mach_port_name_t target_tport = args->target_tport;
1445 int pid = args->pid;
1446 user_addr_t task_addr = args->t;
1447 proc_t p = PROC_NULL;
1448 task_t t1 = TASK_NULL;
1449 task_t task = TASK_NULL;
1450 mach_port_name_t tret = MACH_PORT_NULL;
1451 ipc_port_t tfpport = MACH_PORT_NULL;
1452 ipc_port_t sright = NULL;
1453 int error = 0;
1454 boolean_t is_current_proc = FALSE;
1455 struct proc_ident pident = {0};
1456
1457 AUDIT_MACH_SYSCALL_ENTER(AUE_DBGPORTFORPID);
1458 AUDIT_ARG(pid, pid);
1459 AUDIT_ARG(mach_port1, target_tport);
1460
1461 /* Always check if pid == 0 */
1462 if (pid == 0) {
1463 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
1464 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
1465 return KERN_FAILURE;
1466 }
1467
1468 t1 = port_name_to_task(target_tport);
1469 if (t1 == TASK_NULL) {
1470 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
1471 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
1472 return KERN_FAILURE;
1473 }
1474
1475 p = proc_find(pid);
1476 if (p == PROC_NULL) {
1477 error = KERN_FAILURE;
1478 goto tfpout;
1479 }
1480 pident = proc_ident(p);
1481 is_current_proc = (p == current_proc());
1482
1483 #if CONFIG_AUDIT
1484 AUDIT_ARG(process, p);
1485 #endif
1486
1487 if (!(task_for_pid_posix_check(p))) {
1488 error = KERN_FAILURE;
1489 goto tfpout;
1490 }
1491
1492 if (p->task == TASK_NULL) {
1493 error = KERN_SUCCESS;
1494 goto tfpout;
1495 }
1496
1497 /*
1498 * Grab a task reference and drop the proc reference before making any upcalls.
1499 */
1500 task = p->task;
1501 task_reference(task);
1502
1503 proc_rele(p);
1504 p = PROC_NULL;
1505
1506 if (!IOTaskHasEntitlement(current_task(), DEBUG_PORT_ENTITLEMENT)) {
1507 #if CONFIG_MACF
1508 error = mac_proc_check_get_task(kauth_cred_get(), &pident);
1509 if (error) {
1510 error = KERN_FAILURE;
1511 goto tfpout;
1512 }
1513 #endif
1514
1515 /* If we aren't root and target's task access port is set... */
1516 if (!kauth_cred_issuser(kauth_cred_get()) &&
1517 !is_current_proc &&
1518 (task_get_task_access_port(task, &tfpport) == 0) &&
1519 (tfpport != IPC_PORT_NULL)) {
1520 if (tfpport == IPC_PORT_DEAD) {
1521 error = KERN_PROTECTION_FAILURE;
1522 goto tfpout;
1523 }
1524
1525
1526 /* Call up to the task access server */
1527 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
1528
1529 if (error != MACH_MSG_SUCCESS) {
1530 if (error == MACH_RCV_INTERRUPTED) {
1531 error = KERN_ABORTED;
1532 } else {
1533 error = KERN_FAILURE;
1534 }
1535 goto tfpout;
1536 }
1537 }
1538 }
1539
1540 /* Check if the task has been corpsified */
1541 if (is_corpsetask(task)) {
1542 error = KERN_FAILURE;
1543 goto tfpout;
1544 }
1545
1546 error = task_get_debug_control_port(task, &sright);
1547 if (error != KERN_SUCCESS) {
1548 goto tfpout;
1549 }
1550
1551 tret = ipc_port_copyout_send(
1552 sright,
1553 get_task_ipcspace(current_task()));
1554
1555 error = KERN_SUCCESS;
1556
1557 tfpout:
1558 task_deallocate(t1);
1559 AUDIT_ARG(mach_port2, tret);
1560 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1561
1562 if (tfpport != IPC_PORT_NULL) {
1563 ipc_port_release_send(tfpport);
1564 }
1565 if (task != TASK_NULL) {
1566 task_deallocate(task);
1567 }
1568 if (p != PROC_NULL) {
1569 proc_rele(p);
1570 }
1571 AUDIT_MACH_SYSCALL_EXIT(error);
1572 return error;
1573 }
1574
1575 kern_return_t
1576 pid_resume(struct proc *p __unused, struct pid_resume_args *args, int *ret)
1577 {
1578 task_t target = NULL;
1579 proc_t targetproc = PROC_NULL;
1580 int pid = args->pid;
1581 int error = 0;
1582 mach_port_t tfpport = MACH_PORT_NULL;
1583
1584 if (pid == 0) {
1585 error = EPERM;
1586 goto out;
1587 }
1588
1589 targetproc = proc_find(pid);
1590 if (targetproc == PROC_NULL) {
1591 error = ESRCH;
1592 goto out;
1593 }
1594
1595 if (!task_for_pid_posix_check(targetproc) &&
1596 !IOTaskHasEntitlement(current_task(), PROCESS_RESUME_SUSPEND_ENTITLEMENT)) {
1597 error = EPERM;
1598 goto out;
1599 }
1600
1601 #if CONFIG_MACF
1602 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_RESUME);
1603 if (error) {
1604 error = EPERM;
1605 goto out;
1606 }
1607 #endif
1608
1609 target = targetproc->task;
1610 #ifndef CONFIG_EMBEDDED
1611 if (target != TASK_NULL) {
1612 /* If we aren't root and target's task access port is set... */
1613 if (!kauth_cred_issuser(kauth_cred_get()) &&
1614 targetproc != current_proc() &&
1615 (task_get_task_access_port(target, &tfpport) == 0) &&
1616 (tfpport != IPC_PORT_NULL)) {
1617 if (tfpport == IPC_PORT_DEAD) {
1618 error = EACCES;
1619 goto out;
1620 }
1621
1622 /* Call up to the task access server */
1623 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
1624
1625 if (error != MACH_MSG_SUCCESS) {
1626 if (error == MACH_RCV_INTERRUPTED) {
1627 error = EINTR;
1628 } else {
1629 error = EPERM;
1630 }
1631 goto out;
1632 }
1633 }
1634 }
1635 #endif
1636
1637 #if !XNU_TARGET_OS_OSX
1638 #if SOCKETS
1639 resume_proc_sockets(targetproc);
1640 #endif /* SOCKETS */
1641 #endif /* !XNU_TARGET_OS_OSX */
1642
1643 task_reference(target);
1644
1645 #if CONFIG_MEMORYSTATUS
1646 memorystatus_on_resume(targetproc);
1647 #endif
1648
1649 error = task_pidresume(target);
1650 if (error) {
1651 if (error == KERN_INVALID_ARGUMENT) {
1652 error = EINVAL;
1653 } else {
1654 if (error == KERN_MEMORY_ERROR) {
1655 psignal(targetproc, SIGKILL);
1656 error = EIO;
1657 } else {
1658 error = EPERM;
1659 }
1660 }
1661 }
1662
1663 task_deallocate(target);
1664
1665 out:
1666 if (tfpport != IPC_PORT_NULL) {
1667 ipc_port_release_send(tfpport);
1668 }
1669
1670 if (targetproc != PROC_NULL) {
1671 proc_rele(targetproc);
1672 }
1673
1674 *ret = error;
1675 return error;
1676 }
1677
1678 #if CONFIG_EMBEDDED
1679 /*
1680 * Freeze the specified process (provided in args->pid), or find and freeze a PID.
1681 * When a process is specified, this call is blocking, otherwise we wake up the
1682 * freezer thread and do not block on a process being frozen.
1683 */
1684 kern_return_t
1685 pid_hibernate(struct proc *p __unused, struct pid_hibernate_args *args, int *ret)
1686 {
1687 int error = 0;
1688 proc_t targetproc = PROC_NULL;
1689 int pid = args->pid;
1690
1691 #ifndef CONFIG_FREEZE
1692 #pragma unused(pid)
1693 #else
1694
1695 /*
1696 * If a pid has been provided, we obtain the process handle and call task_for_pid_posix_check().
1697 */
1698
1699 if (pid >= 0) {
1700 targetproc = proc_find(pid);
1701
1702 if (targetproc == PROC_NULL) {
1703 error = ESRCH;
1704 goto out;
1705 }
1706
1707 if (!task_for_pid_posix_check(targetproc)) {
1708 error = EPERM;
1709 goto out;
1710 }
1711 }
1712
1713 #if CONFIG_MACF
1714 //Note that targetproc may be null
1715 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_HIBERNATE);
1716 if (error) {
1717 error = EPERM;
1718 goto out;
1719 }
1720 #endif
1721
1722 if (pid == -2) {
1723 vm_pageout_anonymous_pages();
1724 } else if (pid == -1) {
1725 memorystatus_on_inactivity(targetproc);
1726 } else {
1727 error = memorystatus_freeze_process_sync(targetproc);
1728 }
1729
1730 out:
1731
1732 #endif /* CONFIG_FREEZE */
1733
1734 if (targetproc != PROC_NULL) {
1735 proc_rele(targetproc);
1736 }
1737 *ret = error;
1738 return error;
1739 }
1740 #endif /* CONFIG_EMBEDDED */
1741
1742 #if SOCKETS
1743 int
1744 networking_memstatus_callout(proc_t p, uint32_t status)
1745 {
1746 struct fileproc *fp;
1747
1748 /*
1749 * proc list lock NOT held
1750 * proc lock NOT held
1751 * a reference on the proc has been held / shall be dropped by the caller.
1752 */
1753 LCK_MTX_ASSERT(proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED);
1754 LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED);
1755
1756 proc_fdlock(p);
1757
1758 fdt_foreach(fp, p) {
1759 switch (FILEGLOB_DTYPE(fp->fp_glob)) {
1760 #if NECP
1761 case DTYPE_NETPOLICY:
1762 necp_fd_memstatus(p, status,
1763 (struct necp_fd_data *)fp->fp_glob->fg_data);
1764 break;
1765 #endif /* NECP */
1766 default:
1767 break;
1768 }
1769 }
1770 proc_fdunlock(p);
1771
1772 return 1;
1773 }
1774
1775
1776 static int
1777 networking_defunct_callout(proc_t p, void *arg)
1778 {
1779 struct pid_shutdown_sockets_args *args = arg;
1780 int pid = args->pid;
1781 int level = args->level;
1782 struct fileproc *fp;
1783
1784 proc_fdlock(p);
1785
1786 fdt_foreach(fp, p) {
1787 struct fileglob *fg = fp->fp_glob;
1788
1789 switch (FILEGLOB_DTYPE(fg)) {
1790 case DTYPE_SOCKET: {
1791 struct socket *so = (struct socket *)fg->fg_data;
1792 if (p->p_pid == pid || so->last_pid == pid ||
1793 ((so->so_flags & SOF_DELEGATED) && so->e_pid == pid)) {
1794 /* Call networking stack with socket and level */
1795 (void)socket_defunct(p, so, level);
1796 }
1797 break;
1798 }
1799 #if NECP
1800 case DTYPE_NETPOLICY:
1801 /* first pass: defunct necp and get stats for ntstat */
1802 if (p->p_pid == pid) {
1803 necp_fd_defunct(p,
1804 (struct necp_fd_data *)fg->fg_data);
1805 }
1806 break;
1807 #endif /* NECP */
1808 default:
1809 break;
1810 }
1811 }
1812
1813 proc_fdunlock(p);
1814
1815 return PROC_RETURNED;
1816 }
1817
1818 int
1819 pid_shutdown_sockets(struct proc *p __unused, struct pid_shutdown_sockets_args *args, int *ret)
1820 {
1821 int error = 0;
1822 proc_t targetproc = PROC_NULL;
1823 int pid = args->pid;
1824 int level = args->level;
1825
1826 if (level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_SVC &&
1827 level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL) {
1828 error = EINVAL;
1829 goto out;
1830 }
1831
1832 targetproc = proc_find(pid);
1833 if (targetproc == PROC_NULL) {
1834 error = ESRCH;
1835 goto out;
1836 }
1837
1838 if (!task_for_pid_posix_check(targetproc) &&
1839 !IOTaskHasEntitlement(current_task(), PROCESS_RESUME_SUSPEND_ENTITLEMENT)) {
1840 error = EPERM;
1841 goto out;
1842 }
1843
1844 #if CONFIG_MACF
1845 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_SHUTDOWN_SOCKETS);
1846 if (error) {
1847 error = EPERM;
1848 goto out;
1849 }
1850 #endif
1851
1852 proc_iterate(PROC_ALLPROCLIST | PROC_NOWAITTRANS,
1853 networking_defunct_callout, args, NULL, NULL);
1854
1855 out:
1856 if (targetproc != PROC_NULL) {
1857 proc_rele(targetproc);
1858 }
1859 *ret = error;
1860 return error;
1861 }
1862
1863 #endif /* SOCKETS */
1864
1865 static int
1866 sysctl_settfp_policy(__unused struct sysctl_oid *oidp, void *arg1,
1867 __unused int arg2, struct sysctl_req *req)
1868 {
1869 int error = 0;
1870 int new_value;
1871
1872 error = SYSCTL_OUT(req, arg1, sizeof(int));
1873 if (error || req->newptr == USER_ADDR_NULL) {
1874 return error;
1875 }
1876
1877 if (!kauth_cred_issuser(kauth_cred_get())) {
1878 return EPERM;
1879 }
1880
1881 if ((error = SYSCTL_IN(req, &new_value, sizeof(int)))) {
1882 goto out;
1883 }
1884 if ((new_value == KERN_TFP_POLICY_DENY)
1885 || (new_value == KERN_TFP_POLICY_DEFAULT)) {
1886 tfp_policy = new_value;
1887 } else {
1888 error = EINVAL;
1889 }
1890 out:
1891 return error;
1892 }
1893
1894 #if defined(SECURE_KERNEL)
1895 static int kern_secure_kernel = 1;
1896 #else
1897 static int kern_secure_kernel = 0;
1898 #endif
1899
1900 SYSCTL_INT(_kern, OID_AUTO, secure_kernel, CTLFLAG_RD | CTLFLAG_LOCKED, &kern_secure_kernel, 0, "");
1901
1902 SYSCTL_NODE(_kern, KERN_TFP, tfp, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "tfp");
1903 SYSCTL_PROC(_kern_tfp, KERN_TFP_POLICY, policy, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
1904 &tfp_policy, sizeof(uint32_t), &sysctl_settfp_policy, "I", "policy");
1905
1906 SYSCTL_INT(_vm, OID_AUTO, shared_region_trace_level, CTLFLAG_RW | CTLFLAG_LOCKED,
1907 &shared_region_trace_level, 0, "");
1908 SYSCTL_INT(_vm, OID_AUTO, shared_region_version, CTLFLAG_RD | CTLFLAG_LOCKED,
1909 &shared_region_version, 0, "");
1910 SYSCTL_INT(_vm, OID_AUTO, shared_region_persistence, CTLFLAG_RW | CTLFLAG_LOCKED,
1911 &shared_region_persistence, 0, "");
1912
1913 /*
1914 * shared_region_check_np:
1915 *
1916 * This system call is intended for dyld.
1917 *
1918 * dyld calls this when any process starts to see if the process's shared
1919 * region is already set up and ready to use.
1920 * This call returns the base address of the first mapping in the
1921 * process's shared region's first mapping.
1922 * dyld will then check what's mapped at that address.
1923 *
1924 * If the shared region is empty, dyld will then attempt to map the shared
1925 * cache file in the shared region via the shared_region_map_np() system call.
1926 *
1927 * If something's already mapped in the shared region, dyld will check if it
1928 * matches the shared cache it would like to use for that process.
1929 * If it matches, evrything's ready and the process can proceed and use the
1930 * shared region.
1931 * If it doesn't match, dyld will unmap the shared region and map the shared
1932 * cache into the process's address space via mmap().
1933 *
1934 * ERROR VALUES
1935 * EINVAL no shared region
1936 * ENOMEM shared region is empty
1937 * EFAULT bad address for "start_address"
1938 */
1939 int
1940 shared_region_check_np(
1941 __unused struct proc *p,
1942 struct shared_region_check_np_args *uap,
1943 __unused int *retvalp)
1944 {
1945 vm_shared_region_t shared_region;
1946 mach_vm_offset_t start_address = 0;
1947 int error = 0;
1948 kern_return_t kr;
1949
1950 SHARED_REGION_TRACE_DEBUG(
1951 ("shared_region: %p [%d(%s)] -> check_np(0x%llx)\n",
1952 (void *)VM_KERNEL_ADDRPERM(current_thread()),
1953 p->p_pid, p->p_comm,
1954 (uint64_t)uap->start_address));
1955
1956 /* retrieve the current tasks's shared region */
1957 shared_region = vm_shared_region_get(current_task());
1958 if (shared_region != NULL) {
1959 /* retrieve address of its first mapping... */
1960 kr = vm_shared_region_start_address(shared_region, &start_address);
1961 if (kr != KERN_SUCCESS) {
1962 error = ENOMEM;
1963 } else {
1964 #if __has_feature(ptrauth_calls)
1965 /*
1966 * Remap any section of the shared library that
1967 * has authenticated pointers into private memory.
1968 */
1969 if (vm_shared_region_auth_remap(shared_region) != KERN_SUCCESS) {
1970 error = ENOMEM;
1971 }
1972 #endif /* __has_feature(ptrauth_calls) */
1973
1974 /* ... and give it to the caller */
1975 if (error == 0) {
1976 error = copyout(&start_address,
1977 (user_addr_t) uap->start_address,
1978 sizeof(start_address));
1979 }
1980 if (error != 0) {
1981 SHARED_REGION_TRACE_ERROR(
1982 ("shared_region: %p [%d(%s)] "
1983 "check_np(0x%llx) "
1984 "copyout(0x%llx) error %d\n",
1985 (void *)VM_KERNEL_ADDRPERM(current_thread()),
1986 p->p_pid, p->p_comm,
1987 (uint64_t)uap->start_address, (uint64_t)start_address,
1988 error));
1989 }
1990 }
1991 vm_shared_region_deallocate(shared_region);
1992 } else {
1993 /* no shared region ! */
1994 error = EINVAL;
1995 }
1996
1997 SHARED_REGION_TRACE_DEBUG(
1998 ("shared_region: %p [%d(%s)] check_np(0x%llx) <- 0x%llx %d\n",
1999 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2000 p->p_pid, p->p_comm,
2001 (uint64_t)uap->start_address, (uint64_t)start_address, error));
2002
2003 return error;
2004 }
2005
2006
2007 static int
2008 shared_region_copyin(
2009 struct proc *p,
2010 user_addr_t user_addr,
2011 unsigned int count,
2012 unsigned int element_size,
2013 void *kernel_data)
2014 {
2015 int error = 0;
2016 vm_size_t size = count * element_size;
2017
2018 error = copyin(user_addr, kernel_data, size);
2019 if (error) {
2020 SHARED_REGION_TRACE_ERROR(
2021 ("shared_region: %p [%d(%s)] map(): "
2022 "copyin(0x%llx, %ld) failed (error=%d)\n",
2023 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2024 p->p_pid, p->p_comm,
2025 (uint64_t)user_addr, (long)size, error));
2026 }
2027 return error;
2028 }
2029
2030 #define _SR_FILE_MAPPINGS_MAX_FILES 2
2031
2032 /* forward declaration */
2033 __attribute__((noinline))
2034 static void shared_region_map_and_slide_cleanup(
2035 struct proc *p,
2036 uint32_t files_count,
2037 struct _sr_file_mappings *sr_file_mappings,
2038 struct vm_shared_region *shared_region,
2039 struct vnode *scdir_vp);
2040
2041 /*
2042 * Setup part of _shared_region_map_and_slide().
2043 * It had to be broken out of _shared_region_map_and_slide() to
2044 * prevent compiler inlining from blowing out the stack.
2045 */
2046 __attribute__((noinline))
2047 static int
2048 shared_region_map_and_slide_setup(
2049 struct proc *p,
2050 uint32_t files_count,
2051 struct shared_file_np *files,
2052 uint32_t mappings_count,
2053 struct shared_file_mapping_slide_np *mappings,
2054 struct _sr_file_mappings **sr_file_mappings,
2055 struct vm_shared_region **shared_region_ptr,
2056 struct vnode **scdir_vp,
2057 struct vnode *rdir_vp)
2058 {
2059 int error = 0;
2060 struct _sr_file_mappings *srfmp;
2061 uint32_t mappings_next;
2062 struct vnode_attr va;
2063 off_t fs;
2064 #if CONFIG_MACF
2065 vm_prot_t maxprot = VM_PROT_ALL;
2066 #endif
2067 uint32_t i;
2068 struct vm_shared_region *shared_region;
2069
2070 SHARED_REGION_TRACE_DEBUG(
2071 ("shared_region: %p [%d(%s)] -> map\n",
2072 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2073 p->p_pid, p->p_comm));
2074
2075 if (files_count > _SR_FILE_MAPPINGS_MAX_FILES) {
2076 error = E2BIG;
2077 goto done;
2078 }
2079 if (files_count == 0) {
2080 error = EINVAL;
2081 goto done;
2082 }
2083 *sr_file_mappings = kheap_alloc(KHEAP_TEMP, files_count * sizeof(struct _sr_file_mappings), Z_WAITOK);
2084 if (*sr_file_mappings == NULL) {
2085 error = ENOMEM;
2086 goto done;
2087 }
2088 bzero(*sr_file_mappings, files_count * sizeof(struct _sr_file_mappings));
2089 mappings_next = 0;
2090 for (i = 0; i < files_count; i++) {
2091 srfmp = &(*sr_file_mappings)[i];
2092 srfmp->fd = files[i].sf_fd;
2093 srfmp->mappings_count = files[i].sf_mappings_count;
2094 srfmp->mappings = &mappings[mappings_next];
2095 mappings_next += srfmp->mappings_count;
2096 if (mappings_next > mappings_count) {
2097 error = EINVAL;
2098 goto done;
2099 }
2100 srfmp->slide = files[i].sf_slide;
2101 }
2102
2103 if (scdir_enforce) {
2104 /* get vnode for scdir_path */
2105 error = vnode_lookup(scdir_path, 0, scdir_vp, vfs_context_current());
2106 if (error) {
2107 SHARED_REGION_TRACE_ERROR(
2108 ("shared_region: %p [%d(%s)]: "
2109 "vnode_lookup(%s) failed (error=%d)\n",
2110 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2111 p->p_pid, p->p_comm,
2112 scdir_path, error));
2113 goto done;
2114 }
2115 }
2116
2117 /* get the process's shared region (setup in vm_map_exec()) */
2118 shared_region = vm_shared_region_trim_and_get(current_task());
2119 *shared_region_ptr = shared_region;
2120 if (shared_region == NULL) {
2121 SHARED_REGION_TRACE_ERROR(
2122 ("shared_region: %p [%d(%s)] map(): "
2123 "no shared region\n",
2124 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2125 p->p_pid, p->p_comm));
2126 error = EINVAL;
2127 goto done;
2128 }
2129
2130 /*
2131 * Check the shared region matches the current root
2132 * directory of this process. Deny the mapping to
2133 * avoid tainting the shared region with something that
2134 * doesn't quite belong into it.
2135 */
2136 struct vnode *sr_vnode = vm_shared_region_root_dir(shared_region);
2137 if (sr_vnode != NULL ? rdir_vp != sr_vnode : rdir_vp != rootvnode) {
2138 SHARED_REGION_TRACE_ERROR(
2139 ("shared_region: map(%p) root_dir mismatch\n",
2140 (void *)VM_KERNEL_ADDRPERM(current_thread())));
2141 error = EPERM;
2142 goto done;
2143 }
2144
2145
2146 for (srfmp = &(*sr_file_mappings)[0];
2147 srfmp < &(*sr_file_mappings)[files_count];
2148 srfmp++) {
2149 if (srfmp->mappings_count == 0) {
2150 /* no mappings here... */
2151 continue;
2152 }
2153
2154 /* get file structure from file descriptor */
2155 error = fp_get_ftype(p, srfmp->fd, DTYPE_VNODE, EINVAL, &srfmp->fp);
2156 if (error) {
2157 SHARED_REGION_TRACE_ERROR(
2158 ("shared_region: %p [%d(%s)] map: "
2159 "fd=%d lookup failed (error=%d)\n",
2160 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2161 p->p_pid, p->p_comm, srfmp->fd, error));
2162 goto done;
2163 }
2164
2165 /* we need at least read permission on the file */
2166 if (!(srfmp->fp->fp_glob->fg_flag & FREAD)) {
2167 SHARED_REGION_TRACE_ERROR(
2168 ("shared_region: %p [%d(%s)] map: "
2169 "fd=%d not readable\n",
2170 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2171 p->p_pid, p->p_comm, srfmp->fd));
2172 error = EPERM;
2173 goto done;
2174 }
2175
2176 /* get vnode from file structure */
2177 error = vnode_getwithref((vnode_t) srfmp->fp->fp_glob->fg_data);
2178 if (error) {
2179 SHARED_REGION_TRACE_ERROR(
2180 ("shared_region: %p [%d(%s)] map: "
2181 "fd=%d getwithref failed (error=%d)\n",
2182 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2183 p->p_pid, p->p_comm, srfmp->fd, error));
2184 goto done;
2185 }
2186 srfmp->vp = (struct vnode *) srfmp->fp->fp_glob->fg_data;
2187
2188 /* make sure the vnode is a regular file */
2189 if (srfmp->vp->v_type != VREG) {
2190 SHARED_REGION_TRACE_ERROR(
2191 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2192 "not a file (type=%d)\n",
2193 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2194 p->p_pid, p->p_comm,
2195 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2196 srfmp->vp->v_name, srfmp->vp->v_type));
2197 error = EINVAL;
2198 goto done;
2199 }
2200
2201 #if CONFIG_MACF
2202 /* pass in 0 for the offset argument because AMFI does not need the offset
2203 * of the shared cache */
2204 error = mac_file_check_mmap(vfs_context_ucred(vfs_context_current()),
2205 srfmp->fp->fp_glob, VM_PROT_ALL, MAP_FILE, 0, &maxprot);
2206 if (error) {
2207 goto done;
2208 }
2209 #endif /* MAC */
2210
2211 #if XNU_TARGET_OS_OSX && defined(__arm64__)
2212 /*
2213 * Check if the shared cache is in the trust cache;
2214 * if so, we can skip the root ownership check.
2215 */
2216 #if DEVELOPMENT || DEBUG
2217 /*
2218 * Skip both root ownership and trust cache check if
2219 * enforcement is disabled.
2220 */
2221 if (!cs_system_enforcement()) {
2222 goto after_root_check;
2223 }
2224 #endif /* DEVELOPMENT || DEBUG */
2225 struct cs_blob *blob = csvnode_get_blob(srfmp->vp, 0);
2226 if (blob == NULL) {
2227 SHARED_REGION_TRACE_ERROR(
2228 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2229 "missing CS blob\n",
2230 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2231 p->p_pid, p->p_comm,
2232 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2233 srfmp->vp->v_name));
2234 goto root_check;
2235 }
2236 const uint8_t *cdhash = csblob_get_cdhash(blob);
2237 if (cdhash == NULL) {
2238 SHARED_REGION_TRACE_ERROR(
2239 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2240 "missing cdhash\n",
2241 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2242 p->p_pid, p->p_comm,
2243 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2244 srfmp->vp->v_name));
2245 goto root_check;
2246 }
2247 uint32_t result = pmap_lookup_in_static_trust_cache(cdhash);
2248 boolean_t in_trust_cache = result & (TC_LOOKUP_FOUND << TC_LOOKUP_RESULT_SHIFT);
2249 if (!in_trust_cache) {
2250 SHARED_REGION_TRACE_ERROR(
2251 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2252 "not in trust cache\n",
2253 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2254 p->p_pid, p->p_comm,
2255 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2256 srfmp->vp->v_name));
2257 goto root_check;
2258 }
2259 goto after_root_check;
2260 root_check:
2261 #endif /* XNU_TARGET_OS_OSX && defined(__arm64__) */
2262
2263 /* The shared cache file must be owned by root */
2264 VATTR_INIT(&va);
2265 VATTR_WANTED(&va, va_uid);
2266 error = vnode_getattr(srfmp->vp, &va, vfs_context_current());
2267 if (error) {
2268 SHARED_REGION_TRACE_ERROR(
2269 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2270 "vnode_getattr(%p) failed (error=%d)\n",
2271 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2272 p->p_pid, p->p_comm,
2273 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2274 srfmp->vp->v_name,
2275 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2276 error));
2277 goto done;
2278 }
2279 if (va.va_uid != 0) {
2280 SHARED_REGION_TRACE_ERROR(
2281 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2282 "owned by uid=%d instead of 0\n",
2283 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2284 p->p_pid, p->p_comm,
2285 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2286 srfmp->vp->v_name, va.va_uid));
2287 error = EPERM;
2288 goto done;
2289 }
2290
2291 #if XNU_TARGET_OS_OSX && defined(__arm64__)
2292 after_root_check:
2293 #endif /* XNU_TARGET_OS_OSX && defined(__arm64__) */
2294
2295 #if CONFIG_CSR
2296 if (csr_check(CSR_ALLOW_UNRESTRICTED_FS) != 0) {
2297 VATTR_INIT(&va);
2298 VATTR_WANTED(&va, va_flags);
2299 error = vnode_getattr(srfmp->vp, &va, vfs_context_current());
2300 if (error) {
2301 SHARED_REGION_TRACE_ERROR(
2302 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2303 "vnode_getattr(%p) failed (error=%d)\n",
2304 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2305 p->p_pid, p->p_comm,
2306 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2307 srfmp->vp->v_name,
2308 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2309 error));
2310 goto done;
2311 }
2312
2313 if (!(va.va_flags & SF_RESTRICTED)) {
2314 /*
2315 * CSR is not configured in CSR_ALLOW_UNRESTRICTED_FS mode, and
2316 * the shared cache file is NOT SIP-protected, so reject the
2317 * mapping request
2318 */
2319 SHARED_REGION_TRACE_ERROR(
2320 ("shared_region: %p [%d(%s)] map(%p:'%s'), "
2321 "vnode is not SIP-protected. \n",
2322 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2323 p->p_pid, p->p_comm,
2324 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2325 srfmp->vp->v_name));
2326 error = EPERM;
2327 goto done;
2328 }
2329 }
2330 #else /* CONFIG_CSR */
2331 /* Devices without SIP/ROSP need to make sure that the shared cache is on the root volume. */
2332
2333 assert(rdir_vp != NULL);
2334 if (srfmp->vp->v_mount != rdir_vp->v_mount) {
2335 SHARED_REGION_TRACE_ERROR(
2336 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2337 "not on process's root volume\n",
2338 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2339 p->p_pid, p->p_comm,
2340 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2341 srfmp->vp->v_name));
2342 error = EPERM;
2343 goto done;
2344 }
2345 #endif /* CONFIG_CSR */
2346
2347 if (scdir_enforce) {
2348 /* ensure parent is scdir_vp */
2349 assert(*scdir_vp != NULL);
2350 if (vnode_parent(srfmp->vp) != *scdir_vp) {
2351 SHARED_REGION_TRACE_ERROR(
2352 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2353 "shared cache file not in %s\n",
2354 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2355 p->p_pid, p->p_comm,
2356 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2357 srfmp->vp->v_name, scdir_path));
2358 error = EPERM;
2359 goto done;
2360 }
2361 }
2362
2363 /* get vnode size */
2364 error = vnode_size(srfmp->vp, &fs, vfs_context_current());
2365 if (error) {
2366 SHARED_REGION_TRACE_ERROR(
2367 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2368 "vnode_size(%p) failed (error=%d)\n",
2369 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2370 p->p_pid, p->p_comm,
2371 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2372 srfmp->vp->v_name,
2373 (void *)VM_KERNEL_ADDRPERM(srfmp->vp), error));
2374 goto done;
2375 }
2376 srfmp->file_size = fs;
2377
2378 /* get the file's memory object handle */
2379 srfmp->file_control = ubc_getobject(srfmp->vp, UBC_HOLDOBJECT);
2380 if (srfmp->file_control == MEMORY_OBJECT_CONTROL_NULL) {
2381 SHARED_REGION_TRACE_ERROR(
2382 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2383 "no memory object\n",
2384 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2385 p->p_pid, p->p_comm,
2386 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2387 srfmp->vp->v_name));
2388 error = EINVAL;
2389 goto done;
2390 }
2391
2392 /* check that the mappings are properly covered by code signatures */
2393 if (!cs_system_enforcement()) {
2394 /* code signing is not enforced: no need to check */
2395 } else {
2396 for (i = 0; i < srfmp->mappings_count; i++) {
2397 if (srfmp->mappings[i].sms_init_prot & VM_PROT_ZF) {
2398 /* zero-filled mapping: not backed by the file */
2399 continue;
2400 }
2401 if (ubc_cs_is_range_codesigned(srfmp->vp,
2402 srfmp->mappings[i].sms_file_offset,
2403 srfmp->mappings[i].sms_size)) {
2404 /* this mapping is fully covered by code signatures */
2405 continue;
2406 }
2407 SHARED_REGION_TRACE_ERROR(
2408 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2409 "mapping #%d/%d [0x%llx:0x%llx:0x%llx:0x%x:0x%x] "
2410 "is not code-signed\n",
2411 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2412 p->p_pid, p->p_comm,
2413 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2414 srfmp->vp->v_name,
2415 i, srfmp->mappings_count,
2416 srfmp->mappings[i].sms_address,
2417 srfmp->mappings[i].sms_size,
2418 srfmp->mappings[i].sms_file_offset,
2419 srfmp->mappings[i].sms_max_prot,
2420 srfmp->mappings[i].sms_init_prot));
2421 error = EINVAL;
2422 goto done;
2423 }
2424 }
2425 }
2426 done:
2427 if (error != 0) {
2428 shared_region_map_and_slide_cleanup(p, files_count, *sr_file_mappings, shared_region, *scdir_vp);
2429 *sr_file_mappings = NULL;
2430 *shared_region_ptr = NULL;
2431 *scdir_vp = NULL;
2432 }
2433 return error;
2434 }
2435
2436 /*
2437 * shared_region_map_np()
2438 *
2439 * This system call is intended for dyld.
2440 *
2441 * dyld uses this to map a shared cache file into a shared region.
2442 * This is usually done only the first time a shared cache is needed.
2443 * Subsequent processes will just use the populated shared region without
2444 * requiring any further setup.
2445 */
2446 static int
2447 _shared_region_map_and_slide(
2448 struct proc *p,
2449 uint32_t files_count,
2450 struct shared_file_np *files,
2451 uint32_t mappings_count,
2452 struct shared_file_mapping_slide_np *mappings)
2453 {
2454 int error = 0;
2455 kern_return_t kr = KERN_SUCCESS;
2456 struct _sr_file_mappings *sr_file_mappings = NULL;
2457 struct vnode *scdir_vp = NULL;
2458 struct vnode *rdir_vp = NULL;
2459 struct vm_shared_region *shared_region = NULL;
2460
2461 /*
2462 * Get a reference to the current proc's root dir.
2463 * Need this to prevent racing with chroot.
2464 */
2465 proc_fdlock(p);
2466 rdir_vp = p->p_fd->fd_rdir;
2467 if (rdir_vp == NULL) {
2468 rdir_vp = rootvnode;
2469 }
2470 assert(rdir_vp != NULL);
2471 vnode_get(rdir_vp);
2472 proc_fdunlock(p);
2473
2474 /*
2475 * Turn files, mappings into sr_file_mappings and other setup.
2476 */
2477 error = shared_region_map_and_slide_setup(p, files_count,
2478 files, mappings_count, mappings,
2479 &sr_file_mappings, &shared_region, &scdir_vp, rdir_vp);
2480 if (error != 0) {
2481 vnode_put(rdir_vp);
2482 return error;
2483 }
2484
2485 /* map the file(s) into that shared region's submap */
2486 kr = vm_shared_region_map_file(shared_region, files_count, sr_file_mappings);
2487 if (kr != KERN_SUCCESS) {
2488 SHARED_REGION_TRACE_ERROR(("shared_region: %p [%d(%s)] map(): "
2489 "vm_shared_region_map_file() failed kr=0x%x\n",
2490 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2491 p->p_pid, p->p_comm, kr));
2492 }
2493
2494 /* convert kern_return_t to errno */
2495 switch (kr) {
2496 case KERN_SUCCESS:
2497 error = 0;
2498 break;
2499 case KERN_INVALID_ADDRESS:
2500 error = EFAULT;
2501 break;
2502 case KERN_PROTECTION_FAILURE:
2503 error = EPERM;
2504 break;
2505 case KERN_NO_SPACE:
2506 error = ENOMEM;
2507 break;
2508 case KERN_FAILURE:
2509 case KERN_INVALID_ARGUMENT:
2510 default:
2511 error = EINVAL;
2512 break;
2513 }
2514
2515 /*
2516 * Mark that this process is now using split libraries.
2517 */
2518 if (error == 0 && (p->p_flag & P_NOSHLIB)) {
2519 OSBitAndAtomic(~((uint32_t)P_NOSHLIB), &p->p_flag);
2520 }
2521
2522 vnode_put(rdir_vp);
2523 shared_region_map_and_slide_cleanup(p, files_count, sr_file_mappings, shared_region, scdir_vp);
2524
2525 SHARED_REGION_TRACE_DEBUG(
2526 ("shared_region: %p [%d(%s)] <- map\n",
2527 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2528 p->p_pid, p->p_comm));
2529
2530 return error;
2531 }
2532
2533 /*
2534 * Clean up part of _shared_region_map_and_slide()
2535 * It had to be broken out of _shared_region_map_and_slide() to
2536 * prevent compiler inlining from blowing out the stack.
2537 */
2538 __attribute__((noinline))
2539 static void
2540 shared_region_map_and_slide_cleanup(
2541 struct proc *p,
2542 uint32_t files_count,
2543 struct _sr_file_mappings *sr_file_mappings,
2544 struct vm_shared_region *shared_region,
2545 struct vnode *scdir_vp)
2546 {
2547 struct _sr_file_mappings *srfmp;
2548 struct vnode_attr va;
2549
2550 if (sr_file_mappings != NULL) {
2551 for (srfmp = &sr_file_mappings[0]; srfmp < &sr_file_mappings[files_count]; srfmp++) {
2552 if (srfmp->vp != NULL) {
2553 vnode_lock_spin(srfmp->vp);
2554 srfmp->vp->v_flag |= VSHARED_DYLD;
2555 vnode_unlock(srfmp->vp);
2556
2557 /* update the vnode's access time */
2558 if (!(vnode_vfsvisflags(srfmp->vp) & MNT_NOATIME)) {
2559 VATTR_INIT(&va);
2560 nanotime(&va.va_access_time);
2561 VATTR_SET_ACTIVE(&va, va_access_time);
2562 vnode_setattr(srfmp->vp, &va, vfs_context_current());
2563 }
2564
2565 #if NAMEDSTREAMS
2566 /*
2567 * If the shared cache is compressed, it may
2568 * have a namedstream vnode instantiated for
2569 * for it. That namedstream vnode will also
2570 * have to be marked with VSHARED_DYLD.
2571 */
2572 if (vnode_hasnamedstreams(srfmp->vp)) {
2573 vnode_t svp;
2574 if (vnode_getnamedstream(srfmp->vp, &svp, XATTR_RESOURCEFORK_NAME,
2575 NS_OPEN, 0, vfs_context_kernel()) == 0) {
2576 vnode_lock_spin(svp);
2577 svp->v_flag |= VSHARED_DYLD;
2578 vnode_unlock(svp);
2579 vnode_put(svp);
2580 }
2581 }
2582 #endif /* NAMEDSTREAMS */
2583 /*
2584 * release the vnode...
2585 * ubc_map() still holds it for us in the non-error case
2586 */
2587 (void) vnode_put(srfmp->vp);
2588 srfmp->vp = NULL;
2589 }
2590 if (srfmp->fp != NULL) {
2591 /* release the file descriptor */
2592 fp_drop(p, srfmp->fd, srfmp->fp, 0);
2593 srfmp->fp = NULL;
2594 }
2595 }
2596 kheap_free(KHEAP_TEMP, sr_file_mappings, files_count * sizeof(*sr_file_mappings));
2597 }
2598
2599 if (scdir_vp != NULL) {
2600 (void)vnode_put(scdir_vp);
2601 scdir_vp = NULL;
2602 }
2603
2604 if (shared_region != NULL) {
2605 vm_shared_region_deallocate(shared_region);
2606 }
2607 }
2608
2609
2610 #define SFM_MAX 1024 /* max mapping structs allowed to pass in */
2611
2612 /*
2613 * This interface is used by dyld to map shared caches which are
2614 * for any architecture which doesn't have run time support of pointer
2615 * authentication. Note dyld could also use the new ...map_and_slide_2_np()
2616 * call for this case, however, it just doesn't do that yet.
2617 */
2618 int
2619 shared_region_map_and_slide_np(
2620 struct proc *p,
2621 struct shared_region_map_and_slide_np_args *uap,
2622 __unused int *retvalp)
2623 {
2624 unsigned int mappings_count = uap->count;
2625 unsigned int m;
2626 uint32_t slide = uap->slide;
2627 struct shared_file_np shared_files[1];
2628 struct shared_file_mapping_np legacy_mapping;
2629 struct shared_file_mapping_slide_np *mappings = NULL;
2630 kern_return_t kr = KERN_SUCCESS;
2631
2632 if ((kr = vm_shared_region_sliding_valid(slide)) != KERN_SUCCESS) {
2633 if (kr == KERN_INVALID_ARGUMENT) {
2634 /*
2635 * This will happen if we request sliding again
2636 * with the same slide value that was used earlier
2637 * for the very first sliding.
2638 */
2639 kr = KERN_SUCCESS;
2640 }
2641 goto done;
2642 }
2643
2644 if (mappings_count == 0) {
2645 SHARED_REGION_TRACE_INFO(
2646 ("shared_region: %p [%d(%s)] map(): "
2647 "no mappings\n",
2648 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2649 p->p_pid, p->p_comm));
2650 kr = 0; /* no mappings: we're done ! */
2651 goto done;
2652 } else if (mappings_count <= SFM_MAX) {
2653 mappings = kheap_alloc(KHEAP_TEMP,
2654 mappings_count * sizeof(mappings[0]), Z_WAITOK);
2655 if (mappings == NULL) {
2656 kr = KERN_RESOURCE_SHORTAGE;
2657 goto done;
2658 }
2659 } else {
2660 SHARED_REGION_TRACE_ERROR(
2661 ("shared_region: %p [%d(%s)] map(): "
2662 "too many mappings (%d) max %d\n",
2663 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2664 p->p_pid, p->p_comm,
2665 mappings_count, SFM_MAX));
2666 kr = KERN_FAILURE;
2667 goto done;
2668 }
2669
2670 /*
2671 * Read in the mappings and translate to new format.
2672 */
2673 for (m = 0; m < mappings_count; ++m) {
2674 user_addr_t from_uaddr = uap->mappings + (m * sizeof(struct shared_file_mapping_np));
2675 kr = shared_region_copyin(p, from_uaddr, 1, sizeof(legacy_mapping), &legacy_mapping);
2676 if (kr != 0) {
2677 goto done;
2678 }
2679 mappings[m].sms_address = legacy_mapping.sfm_address;
2680 mappings[m].sms_size = legacy_mapping.sfm_size;
2681 mappings[m].sms_file_offset = legacy_mapping.sfm_file_offset;
2682 mappings[m].sms_max_prot = legacy_mapping.sfm_max_prot;
2683 mappings[m].sms_init_prot = legacy_mapping.sfm_init_prot;
2684 mappings[m].sms_slide_size = uap->slide_size;
2685 mappings[m].sms_slide_start = uap->slide_start;
2686 }
2687
2688 bzero(shared_files, sizeof(shared_files));
2689 shared_files[0].sf_fd = uap->fd;
2690 shared_files[0].sf_mappings_count = mappings_count;
2691 shared_files[0].sf_slide = slide;
2692
2693 kr = _shared_region_map_and_slide(p,
2694 1, /* # of files to map */
2695 &shared_files[0], /* files to map */
2696 mappings_count,
2697 mappings);
2698
2699 done:
2700 if (mappings != NULL) {
2701 kheap_free(KHEAP_TEMP, mappings, mappings_count * sizeof(mappings[0]));
2702 mappings = NULL;
2703 }
2704 return kr;
2705 }
2706
2707 /*
2708 * This interface for setting up shared region mappings is what dyld
2709 * uses for shared caches that have __AUTH sections. All other shared
2710 * caches use the non _2 version.
2711 *
2712 * The slide used for shared regions setup using this interface is done differently
2713 * from the old interface. The slide value passed in the shared_files_np represents
2714 * a max value. The kernel will choose a random value based on that, then use it
2715 * for all shared regions.
2716 */
2717 #define SLIDE_AMOUNT_MASK ~PAGE_MASK
2718
2719 int
2720 shared_region_map_and_slide_2_np(
2721 struct proc *p,
2722 struct shared_region_map_and_slide_2_np_args *uap,
2723 __unused int *retvalp)
2724 {
2725 unsigned int files_count;
2726 struct shared_file_np *shared_files = NULL;
2727 unsigned int mappings_count;
2728 struct shared_file_mapping_slide_np *mappings = NULL;
2729 kern_return_t kr = KERN_SUCCESS;
2730 boolean_t should_slide_mappings = TRUE;
2731
2732 files_count = uap->files_count;
2733 mappings_count = uap->mappings_count;
2734
2735
2736 if (files_count == 0) {
2737 SHARED_REGION_TRACE_INFO(
2738 ("shared_region: %p [%d(%s)] map(): "
2739 "no files\n",
2740 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2741 p->p_pid, p->p_comm));
2742 kr = 0; /* no files to map: we're done ! */
2743 goto done;
2744 } else if (files_count <= _SR_FILE_MAPPINGS_MAX_FILES) {
2745 shared_files = kheap_alloc(KHEAP_TEMP,
2746 files_count * sizeof(shared_files[0]), Z_WAITOK);
2747 if (shared_files == NULL) {
2748 kr = KERN_RESOURCE_SHORTAGE;
2749 goto done;
2750 }
2751 } else {
2752 SHARED_REGION_TRACE_ERROR(
2753 ("shared_region: %p [%d(%s)] map(): "
2754 "too many files (%d) max %d\n",
2755 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2756 p->p_pid, p->p_comm,
2757 files_count, _SR_FILE_MAPPINGS_MAX_FILES));
2758 kr = KERN_FAILURE;
2759 goto done;
2760 }
2761
2762 if (mappings_count == 0) {
2763 SHARED_REGION_TRACE_INFO(
2764 ("shared_region: %p [%d(%s)] map(): "
2765 "no mappings\n",
2766 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2767 p->p_pid, p->p_comm));
2768 kr = 0; /* no mappings: we're done ! */
2769 goto done;
2770 } else if (mappings_count <= SFM_MAX) {
2771 mappings = kheap_alloc(KHEAP_TEMP,
2772 mappings_count * sizeof(mappings[0]), Z_WAITOK);
2773 if (mappings == NULL) {
2774 kr = KERN_RESOURCE_SHORTAGE;
2775 goto done;
2776 }
2777 } else {
2778 SHARED_REGION_TRACE_ERROR(
2779 ("shared_region: %p [%d(%s)] map(): "
2780 "too many mappings (%d) max %d\n",
2781 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2782 p->p_pid, p->p_comm,
2783 mappings_count, SFM_MAX));
2784 kr = KERN_FAILURE;
2785 goto done;
2786 }
2787
2788 kr = shared_region_copyin(p, uap->files, files_count, sizeof(shared_files[0]), shared_files);
2789 if (kr != KERN_SUCCESS) {
2790 goto done;
2791 }
2792
2793 kr = shared_region_copyin(p, uap->mappings, mappings_count, sizeof(mappings[0]), mappings);
2794 if (kr != KERN_SUCCESS) {
2795 goto done;
2796 }
2797
2798 if (should_slide_mappings) {
2799 uint32_t max_slide = shared_files[0].sf_slide;
2800 uint32_t random_val;
2801 uint32_t slide_amount;
2802
2803 if (max_slide != 0) {
2804 read_random(&random_val, sizeof random_val);
2805 slide_amount = ((random_val % max_slide) & SLIDE_AMOUNT_MASK);
2806 } else {
2807 slide_amount = 0;
2808 }
2809
2810 /*
2811 * Fix up the mappings to reflect the desired slide.
2812 */
2813 unsigned int f;
2814 unsigned int m = 0;
2815 unsigned int i;
2816 for (f = 0; f < files_count; ++f) {
2817 shared_files[f].sf_slide = slide_amount;
2818 for (i = 0; i < shared_files[f].sf_mappings_count; ++i, ++m) {
2819 if (m >= mappings_count) {
2820 SHARED_REGION_TRACE_ERROR(
2821 ("shared_region: %p [%d(%s)] map(): "
2822 "mapping count argument was too small\n",
2823 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2824 p->p_pid, p->p_comm));
2825 kr = KERN_FAILURE;
2826 goto done;
2827 }
2828 mappings[m].sms_address += slide_amount;
2829 if (mappings[m].sms_slide_size != 0) {
2830 mappings[i].sms_slide_start += slide_amount;
2831 }
2832 }
2833 }
2834 }
2835 kr = _shared_region_map_and_slide(p, files_count, shared_files, mappings_count, mappings);
2836 done:
2837 if (shared_files != NULL) {
2838 kheap_free(KHEAP_TEMP, shared_files, files_count * sizeof(shared_files[0]));
2839 shared_files = NULL;
2840 }
2841 if (mappings != NULL) {
2842 kheap_free(KHEAP_TEMP, mappings, mappings_count * sizeof(mappings[0]));
2843 mappings = NULL;
2844 }
2845 return kr;
2846 }
2847
2848 /* sysctl overflow room */
2849
2850 SYSCTL_INT(_vm, OID_AUTO, pagesize, CTLFLAG_RD | CTLFLAG_LOCKED,
2851 (int *) &page_size, 0, "vm page size");
2852
2853 /* vm_page_free_target is provided as a makeshift solution for applications that want to
2854 * allocate buffer space, possibly purgeable memory, but not cause inactive pages to be
2855 * reclaimed. It allows the app to calculate how much memory is free outside the free target. */
2856 extern unsigned int vm_page_free_target;
2857 SYSCTL_INT(_vm, OID_AUTO, vm_page_free_target, CTLFLAG_RD | CTLFLAG_LOCKED,
2858 &vm_page_free_target, 0, "Pageout daemon free target");
2859
2860 SYSCTL_INT(_vm, OID_AUTO, memory_pressure, CTLFLAG_RD | CTLFLAG_LOCKED,
2861 &vm_pageout_state.vm_memory_pressure, 0, "Memory pressure indicator");
2862
2863 static int
2864 vm_ctl_page_free_wanted SYSCTL_HANDLER_ARGS
2865 {
2866 #pragma unused(oidp, arg1, arg2)
2867 unsigned int page_free_wanted;
2868
2869 page_free_wanted = mach_vm_ctl_page_free_wanted();
2870 return SYSCTL_OUT(req, &page_free_wanted, sizeof(page_free_wanted));
2871 }
2872 SYSCTL_PROC(_vm, OID_AUTO, page_free_wanted,
2873 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
2874 0, 0, vm_ctl_page_free_wanted, "I", "");
2875
2876 extern unsigned int vm_page_purgeable_count;
2877 SYSCTL_INT(_vm, OID_AUTO, page_purgeable_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2878 &vm_page_purgeable_count, 0, "Purgeable page count");
2879
2880 extern unsigned int vm_page_purgeable_wired_count;
2881 SYSCTL_INT(_vm, OID_AUTO, page_purgeable_wired_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2882 &vm_page_purgeable_wired_count, 0, "Wired purgeable page count");
2883
2884 extern unsigned int vm_page_kern_lpage_count;
2885 SYSCTL_INT(_vm, OID_AUTO, kern_lpage_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2886 &vm_page_kern_lpage_count, 0, "kernel used large pages");
2887
2888 #if DEVELOPMENT || DEBUG
2889 #if __ARM_MIXED_PAGE_SIZE__
2890 static int vm_mixed_pagesize_supported = 1;
2891 #else
2892 static int vm_mixed_pagesize_supported = 0;
2893 #endif /*__ARM_MIXED_PAGE_SIZE__ */
2894 SYSCTL_INT(_debug, OID_AUTO, vm_mixed_pagesize_supported, CTLFLAG_ANYBODY | CTLFLAG_RD | CTLFLAG_LOCKED,
2895 &vm_mixed_pagesize_supported, 0, "kernel support for mixed pagesize");
2896
2897
2898 extern uint64_t get_pages_grabbed_count(void);
2899
2900 static int
2901 pages_grabbed SYSCTL_HANDLER_ARGS
2902 {
2903 #pragma unused(arg1, arg2, oidp)
2904 uint64_t value = get_pages_grabbed_count();
2905 return SYSCTL_OUT(req, &value, sizeof(value));
2906 }
2907
2908 SYSCTL_PROC(_vm, OID_AUTO, pages_grabbed, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED,
2909 0, 0, &pages_grabbed, "QU", "Total pages grabbed");
2910 SYSCTL_ULONG(_vm, OID_AUTO, pages_freed, CTLFLAG_RD | CTLFLAG_LOCKED,
2911 &vm_pageout_vminfo.vm_page_pages_freed, "Total pages freed");
2912
2913 SYSCTL_INT(_vm, OID_AUTO, pageout_purged_objects, CTLFLAG_RD | CTLFLAG_LOCKED,
2914 &vm_pageout_debug.vm_pageout_purged_objects, 0, "System purged object count");
2915 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_busy, CTLFLAG_RD | CTLFLAG_LOCKED,
2916 &vm_pageout_debug.vm_pageout_cleaned_busy, 0, "Cleaned pages busy (deactivated)");
2917 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_nolock, CTLFLAG_RD | CTLFLAG_LOCKED,
2918 &vm_pageout_debug.vm_pageout_cleaned_nolock, 0, "Cleaned pages no-lock (deactivated)");
2919
2920 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_volatile_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2921 &vm_pageout_debug.vm_pageout_cleaned_volatile_reactivated, 0, "Cleaned pages volatile reactivated");
2922 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_fault_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2923 &vm_pageout_debug.vm_pageout_cleaned_fault_reactivated, 0, "Cleaned pages fault reactivated");
2924 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2925 &vm_pageout_debug.vm_pageout_cleaned_reactivated, 0, "Cleaned pages reactivated"); /* sum of all reactivated AND busy and nolock (even though those actually get reDEactivated */
2926 SYSCTL_ULONG(_vm, OID_AUTO, pageout_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED,
2927 &vm_pageout_vminfo.vm_pageout_freed_cleaned, "Cleaned pages freed");
2928 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_reference_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2929 &vm_pageout_debug.vm_pageout_cleaned_reference_reactivated, 0, "Cleaned pages reference reactivated");
2930 SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED,
2931 &vm_pageout_debug.vm_pageout_enqueued_cleaned, 0, ""); /* sum of next two */
2932 #endif /* DEVELOPMENT || DEBUG */
2933
2934 extern int madvise_free_debug;
2935 SYSCTL_INT(_vm, OID_AUTO, madvise_free_debug, CTLFLAG_RW | CTLFLAG_LOCKED,
2936 &madvise_free_debug, 0, "zero-fill on madvise(MADV_FREE*)");
2937
2938 SYSCTL_INT(_vm, OID_AUTO, page_reusable_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2939 &vm_page_stats_reusable.reusable_count, 0, "Reusable page count");
2940 SYSCTL_QUAD(_vm, OID_AUTO, reusable_success, CTLFLAG_RD | CTLFLAG_LOCKED,
2941 &vm_page_stats_reusable.reusable_pages_success, "");
2942 SYSCTL_QUAD(_vm, OID_AUTO, reusable_failure, CTLFLAG_RD | CTLFLAG_LOCKED,
2943 &vm_page_stats_reusable.reusable_pages_failure, "");
2944 SYSCTL_QUAD(_vm, OID_AUTO, reusable_pages_shared, CTLFLAG_RD | CTLFLAG_LOCKED,
2945 &vm_page_stats_reusable.reusable_pages_shared, "");
2946 SYSCTL_QUAD(_vm, OID_AUTO, all_reusable_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2947 &vm_page_stats_reusable.all_reusable_calls, "");
2948 SYSCTL_QUAD(_vm, OID_AUTO, partial_reusable_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2949 &vm_page_stats_reusable.partial_reusable_calls, "");
2950 SYSCTL_QUAD(_vm, OID_AUTO, reuse_success, CTLFLAG_RD | CTLFLAG_LOCKED,
2951 &vm_page_stats_reusable.reuse_pages_success, "");
2952 SYSCTL_QUAD(_vm, OID_AUTO, reuse_failure, CTLFLAG_RD | CTLFLAG_LOCKED,
2953 &vm_page_stats_reusable.reuse_pages_failure, "");
2954 SYSCTL_QUAD(_vm, OID_AUTO, all_reuse_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2955 &vm_page_stats_reusable.all_reuse_calls, "");
2956 SYSCTL_QUAD(_vm, OID_AUTO, partial_reuse_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2957 &vm_page_stats_reusable.partial_reuse_calls, "");
2958 SYSCTL_QUAD(_vm, OID_AUTO, can_reuse_success, CTLFLAG_RD | CTLFLAG_LOCKED,
2959 &vm_page_stats_reusable.can_reuse_success, "");
2960 SYSCTL_QUAD(_vm, OID_AUTO, can_reuse_failure, CTLFLAG_RD | CTLFLAG_LOCKED,
2961 &vm_page_stats_reusable.can_reuse_failure, "");
2962 SYSCTL_QUAD(_vm, OID_AUTO, reusable_reclaimed, CTLFLAG_RD | CTLFLAG_LOCKED,
2963 &vm_page_stats_reusable.reusable_reclaimed, "");
2964 SYSCTL_QUAD(_vm, OID_AUTO, reusable_nonwritable, CTLFLAG_RD | CTLFLAG_LOCKED,
2965 &vm_page_stats_reusable.reusable_nonwritable, "");
2966 SYSCTL_QUAD(_vm, OID_AUTO, reusable_shared, CTLFLAG_RD | CTLFLAG_LOCKED,
2967 &vm_page_stats_reusable.reusable_shared, "");
2968 SYSCTL_QUAD(_vm, OID_AUTO, free_shared, CTLFLAG_RD | CTLFLAG_LOCKED,
2969 &vm_page_stats_reusable.free_shared, "");
2970
2971
2972 extern unsigned int vm_page_free_count, vm_page_speculative_count;
2973 SYSCTL_UINT(_vm, OID_AUTO, page_free_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_free_count, 0, "");
2974 SYSCTL_UINT(_vm, OID_AUTO, page_speculative_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_speculative_count, 0, "");
2975
2976 extern unsigned int vm_page_cleaned_count;
2977 SYSCTL_UINT(_vm, OID_AUTO, page_cleaned_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_cleaned_count, 0, "Cleaned queue size");
2978
2979 extern unsigned int vm_page_pageable_internal_count, vm_page_pageable_external_count;
2980 SYSCTL_UINT(_vm, OID_AUTO, page_pageable_internal_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pageable_internal_count, 0, "");
2981 SYSCTL_UINT(_vm, OID_AUTO, page_pageable_external_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pageable_external_count, 0, "");
2982
2983 /* pageout counts */
2984 SYSCTL_UINT(_vm, OID_AUTO, pageout_inactive_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_state.vm_pageout_inactive_clean, 0, "");
2985 SYSCTL_UINT(_vm, OID_AUTO, pageout_inactive_used, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_state.vm_pageout_inactive_used, 0, "");
2986
2987 SYSCTL_ULONG(_vm, OID_AUTO, pageout_inactive_dirty_internal, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_inactive_dirty_internal, "");
2988 SYSCTL_ULONG(_vm, OID_AUTO, pageout_inactive_dirty_external, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_inactive_dirty_external, "");
2989 SYSCTL_ULONG(_vm, OID_AUTO, pageout_speculative_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_speculative, "");
2990 SYSCTL_ULONG(_vm, OID_AUTO, pageout_freed_external, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_external, "");
2991 SYSCTL_ULONG(_vm, OID_AUTO, pageout_freed_speculative, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_speculative, "");
2992 SYSCTL_ULONG(_vm, OID_AUTO, pageout_freed_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_cleaned, "");
2993
2994
2995 /* counts of pages prefaulted when entering a memory object */
2996 extern int64_t vm_prefault_nb_pages, vm_prefault_nb_bailout;
2997 SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_pages, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_pages, "");
2998 SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_bailout, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_bailout, "");
2999
3000 #if defined (__x86_64__)
3001 extern unsigned int vm_clump_promote_threshold;
3002 SYSCTL_UINT(_vm, OID_AUTO, vm_clump_promote_threshold, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_clump_promote_threshold, 0, "clump size threshold for promotes");
3003 #if DEVELOPMENT || DEBUG
3004 extern unsigned long vm_clump_stats[];
3005 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats1, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[1], "free page allocations from clump of 1 page");
3006 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats2, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[2], "free page allocations from clump of 2 pages");
3007 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats3, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[3], "free page allocations from clump of 3 pages");
3008 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats4, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[4], "free page allocations from clump of 4 pages");
3009 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats5, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[5], "free page allocations from clump of 5 pages");
3010 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats6, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[6], "free page allocations from clump of 6 pages");
3011 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats7, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[7], "free page allocations from clump of 7 pages");
3012 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats8, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[8], "free page allocations from clump of 8 pages");
3013 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats9, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[9], "free page allocations from clump of 9 pages");
3014 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats10, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[10], "free page allocations from clump of 10 pages");
3015 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats11, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[11], "free page allocations from clump of 11 pages");
3016 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats12, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[12], "free page allocations from clump of 12 pages");
3017 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats13, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[13], "free page allocations from clump of 13 pages");
3018 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats14, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[14], "free page allocations from clump of 14 pages");
3019 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats15, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[15], "free page allocations from clump of 15 pages");
3020 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats16, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[16], "free page allocations from clump of 16 pages");
3021 extern unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes;
3022 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_alloc, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_allocs, "free page allocations");
3023 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_inserts, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_inserts, "free page insertions");
3024 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_inrange, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_inrange, "free page insertions that are part of vm_pages");
3025 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_promotes, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_promotes, "pages promoted to head");
3026 #endif /* if DEVELOPMENT || DEBUG */
3027 #endif /* #if defined (__x86_64__) */
3028
3029 #if CONFIG_SECLUDED_MEMORY
3030
3031 SYSCTL_UINT(_vm, OID_AUTO, num_tasks_can_use_secluded_mem, CTLFLAG_RD | CTLFLAG_LOCKED, &num_tasks_can_use_secluded_mem, 0, "");
3032 extern unsigned int vm_page_secluded_target;
3033 extern unsigned int vm_page_secluded_count;
3034 extern unsigned int vm_page_secluded_count_free;
3035 extern unsigned int vm_page_secluded_count_inuse;
3036 extern unsigned int vm_page_secluded_count_over_target;
3037 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_target, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_target, 0, "");
3038 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count, 0, "");
3039 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count_free, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count_free, 0, "");
3040 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count_inuse, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count_inuse, 0, "");
3041 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count_over_target, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count_over_target, 0, "");
3042
3043 extern struct vm_page_secluded_data vm_page_secluded;
3044 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_eligible, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.eligible_for_secluded, 0, "");
3045 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_success_free, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_success_free, 0, "");
3046 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_success_other, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_success_other, 0, "");
3047 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_locked, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_failure_locked, 0, "");
3048 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_state, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_failure_state, 0, "");
3049 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_dirty, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_failure_dirty, 0, "");
3050 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_for_iokit, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_for_iokit, 0, "");
3051 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_for_iokit_success, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_for_iokit_success, 0, "");
3052
3053 #endif /* CONFIG_SECLUDED_MEMORY */
3054
3055 #include <kern/thread.h>
3056 #include <sys/user.h>
3057
3058 void vm_pageout_io_throttle(void);
3059
3060 void
3061 vm_pageout_io_throttle(void)
3062 {
3063 struct uthread *uthread = get_bsdthread_info(current_thread());
3064
3065 /*
3066 * thread is marked as a low priority I/O type
3067 * and the I/O we issued while in this cleaning operation
3068 * collided with normal I/O operations... we'll
3069 * delay in order to mitigate the impact of this
3070 * task on the normal operation of the system
3071 */
3072
3073 if (uthread->uu_lowpri_window) {
3074 throttle_lowpri_io(1);
3075 }
3076 }
3077
3078 int
3079 vm_pressure_monitor(
3080 __unused struct proc *p,
3081 struct vm_pressure_monitor_args *uap,
3082 int *retval)
3083 {
3084 kern_return_t kr;
3085 uint32_t pages_reclaimed;
3086 uint32_t pages_wanted;
3087
3088 kr = mach_vm_pressure_monitor(
3089 (boolean_t) uap->wait_for_pressure,
3090 uap->nsecs_monitored,
3091 (uap->pages_reclaimed) ? &pages_reclaimed : NULL,
3092 &pages_wanted);
3093
3094 switch (kr) {
3095 case KERN_SUCCESS:
3096 break;
3097 case KERN_ABORTED:
3098 return EINTR;
3099 default:
3100 return EINVAL;
3101 }
3102
3103 if (uap->pages_reclaimed) {
3104 if (copyout((void *)&pages_reclaimed,
3105 uap->pages_reclaimed,
3106 sizeof(pages_reclaimed)) != 0) {
3107 return EFAULT;
3108 }
3109 }
3110
3111 *retval = (int) pages_wanted;
3112 return 0;
3113 }
3114
3115 int
3116 kas_info(struct proc *p,
3117 struct kas_info_args *uap,
3118 int *retval __unused)
3119 {
3120 #ifndef CONFIG_KAS_INFO
3121 (void)p;
3122 (void)uap;
3123 return ENOTSUP;
3124 #else /* CONFIG_KAS_INFO */
3125 int selector = uap->selector;
3126 user_addr_t valuep = uap->value;
3127 user_addr_t sizep = uap->size;
3128 user_size_t size, rsize;
3129 int error;
3130
3131 if (!kauth_cred_issuser(kauth_cred_get())) {
3132 return EPERM;
3133 }
3134
3135 #if CONFIG_MACF
3136 error = mac_system_check_kas_info(kauth_cred_get(), selector);
3137 if (error) {
3138 return error;
3139 }
3140 #endif
3141
3142 if (IS_64BIT_PROCESS(p)) {
3143 user64_size_t size64;
3144 error = copyin(sizep, &size64, sizeof(size64));
3145 size = (user_size_t)size64;
3146 } else {
3147 user32_size_t size32;
3148 error = copyin(sizep, &size32, sizeof(size32));
3149 size = (user_size_t)size32;
3150 }
3151 if (error) {
3152 return error;
3153 }
3154
3155 switch (selector) {
3156 case KAS_INFO_KERNEL_TEXT_SLIDE_SELECTOR:
3157 {
3158 uint64_t slide = vm_kernel_slide;
3159
3160 if (sizeof(slide) != size) {
3161 return EINVAL;
3162 }
3163
3164 error = copyout(&slide, valuep, sizeof(slide));
3165 if (error) {
3166 return error;
3167 }
3168 rsize = size;
3169 }
3170 break;
3171 case KAS_INFO_KERNEL_SEGMENT_VMADDR_SELECTOR:
3172 {
3173 uint32_t i;
3174 kernel_mach_header_t *mh = &_mh_execute_header;
3175 struct load_command *cmd;
3176 cmd = (struct load_command*) &mh[1];
3177 uint64_t *bases;
3178 rsize = mh->ncmds * sizeof(uint64_t);
3179
3180 /*
3181 * Return the size if no data was passed
3182 */
3183 if (valuep == 0) {
3184 break;
3185 }
3186
3187 if (rsize > size) {
3188 return EINVAL;
3189 }
3190
3191 bases = kheap_alloc(KHEAP_TEMP, rsize, Z_WAITOK | Z_ZERO);
3192
3193 for (i = 0; i < mh->ncmds; i++) {
3194 if (cmd->cmd == LC_SEGMENT_KERNEL) {
3195 __IGNORE_WCASTALIGN(kernel_segment_command_t * sg = (kernel_segment_command_t *) cmd);
3196 bases[i] = (uint64_t)sg->vmaddr;
3197 }
3198 cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
3199 }
3200
3201 error = copyout(bases, valuep, rsize);
3202
3203 kheap_free(KHEAP_TEMP, bases, rsize);
3204
3205 if (error) {
3206 return error;
3207 }
3208 }
3209 break;
3210 default:
3211 return EINVAL;
3212 }
3213
3214 if (IS_64BIT_PROCESS(p)) {
3215 user64_size_t size64 = (user64_size_t)rsize;
3216 error = copyout(&size64, sizep, sizeof(size64));
3217 } else {
3218 user32_size_t size32 = (user32_size_t)rsize;
3219 error = copyout(&size32, sizep, sizeof(size32));
3220 }
3221
3222 return error;
3223 #endif /* CONFIG_KAS_INFO */
3224 }
3225
3226 #if __has_feature(ptrauth_calls)
3227 /*
3228 * Generate a random pointer signing key that isn't 0.
3229 */
3230 uint64_t
3231 generate_jop_key(void)
3232 {
3233 uint64_t key;
3234
3235 do {
3236 read_random(&key, sizeof key);
3237 } while (key == 0);
3238 return key;
3239 }
3240 #endif /* __has_feature(ptrauth_calls) */
3241
3242
3243 #pragma clang diagnostic push
3244 #pragma clang diagnostic ignored "-Wcast-qual"
3245 #pragma clang diagnostic ignored "-Wunused-function"
3246
3247 static void
3248 asserts()
3249 {
3250 static_assert(sizeof(vm_min_kernel_address) == sizeof(unsigned long));
3251 static_assert(sizeof(vm_max_kernel_address) == sizeof(unsigned long));
3252 }
3253
3254 SYSCTL_ULONG(_vm, OID_AUTO, vm_min_kernel_address, CTLFLAG_RD, (unsigned long *) &vm_min_kernel_address, "");
3255 SYSCTL_ULONG(_vm, OID_AUTO, vm_max_kernel_address, CTLFLAG_RD, (unsigned long *) &vm_max_kernel_address, "");
3256 #pragma clang diagnostic pop
3257
3258 extern uint32_t vm_page_pages;
3259 SYSCTL_UINT(_vm, OID_AUTO, pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pages, 0, "");
3260
3261 extern uint32_t vm_page_busy_absent_skipped;
3262 SYSCTL_UINT(_vm, OID_AUTO, page_busy_absent_skipped, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_busy_absent_skipped, 0, "");
3263
3264 extern uint32_t vm_page_upl_tainted;
3265 SYSCTL_UINT(_vm, OID_AUTO, upl_pages_tainted, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_upl_tainted, 0, "");
3266
3267 extern uint32_t vm_page_iopl_tainted;
3268 SYSCTL_UINT(_vm, OID_AUTO, iopl_pages_tainted, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_iopl_tainted, 0, "");
3269
3270 #if (__arm__ || __arm64__) && (DEVELOPMENT || DEBUG)
3271 extern int vm_footprint_suspend_allowed;
3272 SYSCTL_INT(_vm, OID_AUTO, footprint_suspend_allowed, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_footprint_suspend_allowed, 0, "");
3273
3274 extern void pmap_footprint_suspend(vm_map_t map, boolean_t suspend);
3275 static int
3276 sysctl_vm_footprint_suspend SYSCTL_HANDLER_ARGS
3277 {
3278 #pragma unused(oidp, arg1, arg2)
3279 int error = 0;
3280 int new_value;
3281
3282 if (req->newptr == USER_ADDR_NULL) {
3283 return 0;
3284 }
3285 error = SYSCTL_IN(req, &new_value, sizeof(int));
3286 if (error) {
3287 return error;
3288 }
3289 if (!vm_footprint_suspend_allowed) {
3290 if (new_value != 0) {
3291 /* suspends are not allowed... */
3292 return 0;
3293 }
3294 /* ... but let resumes proceed */
3295 }
3296 DTRACE_VM2(footprint_suspend,
3297 vm_map_t, current_map(),
3298 int, new_value);
3299
3300 pmap_footprint_suspend(current_map(), new_value);
3301
3302 return 0;
3303 }
3304 SYSCTL_PROC(_vm, OID_AUTO, footprint_suspend,
3305 CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_LOCKED | CTLFLAG_MASKED,
3306 0, 0, &sysctl_vm_footprint_suspend, "I", "");
3307 #endif /* (__arm__ || __arm64__) && (DEVELOPMENT || DEBUG) */
3308
3309 extern uint64_t vm_map_corpse_footprint_count;
3310 extern uint64_t vm_map_corpse_footprint_size_avg;
3311 extern uint64_t vm_map_corpse_footprint_size_max;
3312 extern uint64_t vm_map_corpse_footprint_full;
3313 extern uint64_t vm_map_corpse_footprint_no_buf;
3314 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_count,
3315 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_count, "");
3316 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_size_avg,
3317 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_size_avg, "");
3318 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_size_max,
3319 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_size_max, "");
3320 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_full,
3321 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_full, "");
3322 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_no_buf,
3323 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_no_buf, "");
3324
3325
3326 extern uint64_t shared_region_pager_copied;
3327 extern uint64_t shared_region_pager_slid;
3328 extern uint64_t shared_region_pager_slid_error;
3329 extern uint64_t shared_region_pager_reclaimed;
3330 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_copied,
3331 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_copied, "");
3332 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_slid,
3333 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_slid, "");
3334 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_slid_error,
3335 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_slid_error, "");
3336 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_reclaimed,
3337 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_reclaimed, "");
3338 extern int shared_region_destroy_delay;
3339 SYSCTL_INT(_vm, OID_AUTO, shared_region_destroy_delay,
3340 CTLFLAG_RW | CTLFLAG_LOCKED, &shared_region_destroy_delay, 0, "");
3341
3342 #if MACH_ASSERT
3343 extern int pmap_ledgers_panic_leeway;
3344 SYSCTL_INT(_vm, OID_AUTO, pmap_ledgers_panic_leeway, CTLFLAG_RW | CTLFLAG_LOCKED, &pmap_ledgers_panic_leeway, 0, "");
3345 #endif /* MACH_ASSERT */
3346
3347 extern int vm_protect_privileged_from_untrusted;
3348 SYSCTL_INT(_vm, OID_AUTO, protect_privileged_from_untrusted,
3349 CTLFLAG_RW | CTLFLAG_LOCKED, &vm_protect_privileged_from_untrusted, 0, "");
3350 extern uint64_t vm_copied_on_read;
3351 SYSCTL_QUAD(_vm, OID_AUTO, copied_on_read,
3352 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_copied_on_read, "");
3353
3354 extern int vm_shared_region_count;
3355 extern int vm_shared_region_peak;
3356 SYSCTL_INT(_vm, OID_AUTO, shared_region_count,
3357 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_shared_region_count, 0, "");
3358 SYSCTL_INT(_vm, OID_AUTO, shared_region_peak,
3359 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_shared_region_peak, 0, "");
3360 #if DEVELOPMENT || DEBUG
3361 extern unsigned int shared_region_pagers_resident_count;
3362 SYSCTL_INT(_vm, OID_AUTO, shared_region_pagers_resident_count,
3363 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pagers_resident_count, 0, "");
3364 extern unsigned int shared_region_pagers_resident_peak;
3365 SYSCTL_INT(_vm, OID_AUTO, shared_region_pagers_resident_peak,
3366 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pagers_resident_peak, 0, "");
3367 extern int shared_region_pager_count;
3368 SYSCTL_INT(_vm, OID_AUTO, shared_region_pager_count,
3369 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_count, 0, "");
3370 #if __has_feature(ptrauth_calls)
3371 extern int shared_region_key_count;
3372 SYSCTL_INT(_vm, OID_AUTO, shared_region_key_count,
3373 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_key_count, 0, "");
3374 extern int vm_shared_region_reslide_count;
3375 SYSCTL_INT(_vm, OID_AUTO, shared_region_reslide_count,
3376 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_shared_region_reslide_count, 0, "");
3377 #endif /* __has_feature(ptrauth_calls) */
3378 #endif /* DEVELOPMENT || DEBUG */
3379
3380 #if MACH_ASSERT
3381 extern int debug4k_filter;
3382 SYSCTL_INT(_vm, OID_AUTO, debug4k_filter, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_filter, 0, "");
3383 extern int debug4k_panic_on_terminate;
3384 SYSCTL_INT(_vm, OID_AUTO, debug4k_panic_on_terminate, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_panic_on_terminate, 0, "");
3385 extern int debug4k_panic_on_exception;
3386 SYSCTL_INT(_vm, OID_AUTO, debug4k_panic_on_exception, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_panic_on_exception, 0, "");
3387 extern int debug4k_panic_on_misaligned_sharing;
3388 SYSCTL_INT(_vm, OID_AUTO, debug4k_panic_on_misaligned_sharing, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_panic_on_misaligned_sharing, 0, "");
3389 #endif /* MACH_ASSERT */
3390
3391 /*
3392 * A sysctl which causes all existing shared regions to become stale. They
3393 * will no longer be used by anything new and will be torn down as soon as
3394 * the last existing user exits. A write of non-zero value causes that to happen.
3395 * This should only be used by launchd, so we check that this is initproc.
3396 */
3397 static int
3398 shared_region_pivot(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
3399 {
3400 unsigned int value = 0;
3401 int changed = 0;
3402 int error = sysctl_io_number(req, 0, sizeof(value), &value, &changed);
3403 if (error || !changed) {
3404 return error;
3405 }
3406 if (current_proc() != initproc) {
3407 return EPERM;
3408 }
3409
3410 vm_shared_region_pivot();
3411
3412 return 0;
3413 }
3414
3415 SYSCTL_PROC(_vm, OID_AUTO, shared_region_pivot,
3416 CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED,
3417 0, 0, shared_region_pivot, "I", "");
3418
3419 extern int vm_remap_old_path, vm_remap_new_path;
3420 SYSCTL_INT(_vm, OID_AUTO, remap_old_path,
3421 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_remap_old_path, 0, "");
3422 SYSCTL_INT(_vm, OID_AUTO, remap_new_path,
3423 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_remap_new_path, 0, "");
mirror of XNU