]> git.saurik.com Git - apple/xnu.git/blob - bsd/vm/vm_unix.c
projects / apple / xnu.git / blob
? search:


9d8c3820995c2c67d87a1d54d2f9283d4b30be15
[apple/xnu.git] / bsd / vm / vm_unix.c
1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Mach Operating System
30 * Copyright (c) 1987 Carnegie-Mellon University
31 * All rights reserved. The CMU software License Agreement specifies
32 * the terms and conditions for use and redistribution.
33 */
34 /*
35 * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
36 * support for mandatory and extensible security protections. This notice
37 * is included in support of clause 2.2 (b) of the Apple Public License,
38 * Version 2.0.
39 */
40 #include <vm/vm_options.h>
41
42 #include <kern/task.h>
43 #include <kern/thread.h>
44 #include <kern/debug.h>
45 #include <kern/extmod_statistics.h>
46 #include <mach/mach_traps.h>
47 #include <mach/port.h>
48 #include <mach/sdt.h>
49 #include <mach/task.h>
50 #include <mach/task_access.h>
51 #include <mach/task_special_ports.h>
52 #include <mach/time_value.h>
53 #include <mach/vm_map.h>
54 #include <mach/vm_param.h>
55 #include <mach/vm_prot.h>
56 #include <machine/machine_routines.h>
57
58 #include <sys/file_internal.h>
59 #include <sys/param.h>
60 #include <sys/systm.h>
61 #include <sys/dir.h>
62 #include <sys/namei.h>
63 #include <sys/proc_internal.h>
64 #include <sys/kauth.h>
65 #include <sys/vm.h>
66 #include <sys/file.h>
67 #include <sys/vnode_internal.h>
68 #include <sys/mount.h>
69 #include <sys/xattr.h>
70 #include <sys/trace.h>
71 #include <sys/kernel.h>
72 #include <sys/ubc_internal.h>
73 #include <sys/user.h>
74 #include <sys/syslog.h>
75 #include <sys/stat.h>
76 #include <sys/sysproto.h>
77 #include <sys/mman.h>
78 #include <sys/sysctl.h>
79 #include <sys/cprotect.h>
80 #include <sys/kpi_socket.h>
81 #include <sys/kas_info.h>
82 #include <sys/socket.h>
83 #include <sys/socketvar.h>
84 #include <sys/random.h>
85 #if NECP
86 #include <net/necp.h>
87 #endif /* NECP */
88
89 #include <security/audit/audit.h>
90 #include <security/mac.h>
91 #include <bsm/audit_kevents.h>
92
93 #include <kern/kalloc.h>
94 #include <vm/vm_map.h>
95 #include <vm/vm_kern.h>
96 #include <vm/vm_pageout.h>
97
98 #include <mach/shared_region.h>
99 #include <vm/vm_shared_region.h>
100
101 #include <vm/vm_protos.h>
102
103 #include <sys/kern_memorystatus.h>
104 #include <sys/kern_memorystatus_freeze.h>
105 #include <sys/proc_internal.h>
106
107 #if CONFIG_MACF
108 #include <security/mac_framework.h>
109 #endif
110
111 #include <kern/bits.h>
112
113 #if CONFIG_CSR
114 #include <sys/csr.h>
115 #endif /* CONFIG_CSR */
116 #include <IOKit/IOBSD.h>
117
118 #if VM_MAP_DEBUG_APPLE_PROTECT
119 SYSCTL_INT(_vm, OID_AUTO, map_debug_apple_protect, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_map_debug_apple_protect, 0, "");
120 #endif /* VM_MAP_DEBUG_APPLE_PROTECT */
121
122 #if VM_MAP_DEBUG_FOURK
123 SYSCTL_INT(_vm, OID_AUTO, map_debug_fourk, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_map_debug_fourk, 0, "");
124 #endif /* VM_MAP_DEBUG_FOURK */
125
126 #if DEVELOPMENT || DEBUG
127
128 static int
129 sysctl_kmem_alloc_contig SYSCTL_HANDLER_ARGS
130 {
131 #pragma unused(arg1, arg2)
132 vm_offset_t kaddr;
133 kern_return_t kr;
134 int error = 0;
135 int size = 0;
136
137 error = sysctl_handle_int(oidp, &size, 0, req);
138 if (error || !req->newptr) {
139 return error;
140 }
141
142 kr = kmem_alloc_contig(kernel_map, &kaddr, (vm_size_t)size, 0, 0, 0, 0, VM_KERN_MEMORY_IOKIT);
143
144 if (kr == KERN_SUCCESS) {
145 kmem_free(kernel_map, kaddr, size);
146 }
147
148 return error;
149 }
150
151 SYSCTL_PROC(_vm, OID_AUTO, kmem_alloc_contig, CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED,
152 0, 0, &sysctl_kmem_alloc_contig, "I", "");
153
154 extern int vm_region_footprint;
155 SYSCTL_INT(_vm, OID_AUTO, region_footprint, CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_LOCKED, &vm_region_footprint, 0, "");
156
157 #endif /* DEVELOPMENT || DEBUG */
158
159 static int
160 sysctl_vm_self_region_footprint SYSCTL_HANDLER_ARGS
161 {
162 #pragma unused(arg1, arg2, oidp)
163 int error = 0;
164 int value;
165
166 value = task_self_region_footprint();
167 error = SYSCTL_OUT(req, &value, sizeof(int));
168 if (error) {
169 return error;
170 }
171
172 if (!req->newptr) {
173 return 0;
174 }
175
176 error = SYSCTL_IN(req, &value, sizeof(int));
177 if (error) {
178 return error;
179 }
180 task_self_region_footprint_set(value);
181 return 0;
182 }
183 SYSCTL_PROC(_vm, OID_AUTO, self_region_footprint, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_LOCKED | CTLFLAG_MASKED, 0, 0, &sysctl_vm_self_region_footprint, "I", "");
184
185 static int
186 sysctl_vm_self_region_page_size SYSCTL_HANDLER_ARGS
187 {
188 #pragma unused(arg1, arg2, oidp)
189 int error = 0;
190 int value;
191
192 value = (1 << thread_self_region_page_shift());
193 error = SYSCTL_OUT(req, &value, sizeof(int));
194 if (error) {
195 return error;
196 }
197
198 if (!req->newptr) {
199 return 0;
200 }
201
202 error = SYSCTL_IN(req, &value, sizeof(int));
203 if (error) {
204 return error;
205 }
206
207 if (value != 0 && value != 4096 && value != 16384) {
208 return EINVAL;
209 }
210
211 #if !__ARM_MIXED_PAGE_SIZE__
212 if (value != vm_map_page_size(current_map())) {
213 return EINVAL;
214 }
215 #endif /* !__ARM_MIXED_PAGE_SIZE__ */
216
217 thread_self_region_page_shift_set(bit_first(value));
218 return 0;
219 }
220 SYSCTL_PROC(_vm, OID_AUTO, self_region_page_size, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_LOCKED | CTLFLAG_MASKED, 0, 0, &sysctl_vm_self_region_page_size, "I", "");
221
222
223 #if DEVELOPMENT || DEBUG
224 extern int panic_on_unsigned_execute;
225 SYSCTL_INT(_vm, OID_AUTO, panic_on_unsigned_execute, CTLFLAG_RW | CTLFLAG_LOCKED, &panic_on_unsigned_execute, 0, "");
226 #endif /* DEVELOPMENT || DEBUG */
227
228 extern int cs_executable_create_upl;
229 extern int cs_executable_wire;
230 SYSCTL_INT(_vm, OID_AUTO, cs_executable_create_upl, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_create_upl, 0, "");
231 SYSCTL_INT(_vm, OID_AUTO, cs_executable_wire, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_wire, 0, "");
232
233 #if DEVELOPMENT || DEBUG
234 extern int radar_20146450;
235 SYSCTL_INT(_vm, OID_AUTO, radar_20146450, CTLFLAG_RW | CTLFLAG_LOCKED, &radar_20146450, 0, "");
236
237 extern int macho_printf;
238 SYSCTL_INT(_vm, OID_AUTO, macho_printf, CTLFLAG_RW | CTLFLAG_LOCKED, &macho_printf, 0, "");
239
240 extern int apple_protect_pager_data_request_debug;
241 SYSCTL_INT(_vm, OID_AUTO, apple_protect_pager_data_request_debug, CTLFLAG_RW | CTLFLAG_LOCKED, &apple_protect_pager_data_request_debug, 0, "");
242
243 #if __arm__ || __arm64__
244 /* These are meant to support the page table accounting unit test. */
245 extern unsigned int arm_hardware_page_size;
246 extern unsigned int arm_pt_desc_size;
247 extern unsigned int arm_pt_root_size;
248 extern unsigned int free_page_size_tt_count;
249 extern unsigned int free_two_page_size_tt_count;
250 extern unsigned int free_tt_count;
251 extern unsigned int inuse_user_tteroot_count;
252 extern unsigned int inuse_kernel_tteroot_count;
253 extern unsigned int inuse_user_ttepages_count;
254 extern unsigned int inuse_kernel_ttepages_count;
255 extern unsigned int inuse_user_ptepages_count;
256 extern unsigned int inuse_kernel_ptepages_count;
257 SYSCTL_UINT(_vm, OID_AUTO, native_hw_pagesize, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_hardware_page_size, 0, "");
258 SYSCTL_UINT(_vm, OID_AUTO, arm_pt_desc_size, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_pt_desc_size, 0, "");
259 SYSCTL_UINT(_vm, OID_AUTO, arm_pt_root_size, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_pt_root_size, 0, "");
260 SYSCTL_UINT(_vm, OID_AUTO, free_1page_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_page_size_tt_count, 0, "");
261 SYSCTL_UINT(_vm, OID_AUTO, free_2page_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_two_page_size_tt_count, 0, "");
262 SYSCTL_UINT(_vm, OID_AUTO, free_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_tt_count, 0, "");
263 SYSCTL_UINT(_vm, OID_AUTO, user_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_tteroot_count, 0, "");
264 SYSCTL_UINT(_vm, OID_AUTO, kernel_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_tteroot_count, 0, "");
265 SYSCTL_UINT(_vm, OID_AUTO, user_tte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_ttepages_count, 0, "");
266 SYSCTL_UINT(_vm, OID_AUTO, kernel_tte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_ttepages_count, 0, "");
267 SYSCTL_UINT(_vm, OID_AUTO, user_pte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_ptepages_count, 0, "");
268 SYSCTL_UINT(_vm, OID_AUTO, kernel_pte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_ptepages_count, 0, "");
269 #if DEVELOPMENT || DEBUG
270 extern unsigned long pmap_asid_flushes;
271 SYSCTL_ULONG(_vm, OID_AUTO, pmap_asid_flushes, CTLFLAG_RD | CTLFLAG_LOCKED, &pmap_asid_flushes, "");
272 extern unsigned long pmap_asid_hits;
273 SYSCTL_ULONG(_vm, OID_AUTO, pmap_asid_hits, CTLFLAG_RD | CTLFLAG_LOCKED, &pmap_asid_hits, "");
274 extern unsigned long pmap_asid_misses;
275 SYSCTL_ULONG(_vm, OID_AUTO, pmap_asid_misses, CTLFLAG_RD | CTLFLAG_LOCKED, &pmap_asid_misses, "");
276 #endif
277 #endif /* __arm__ || __arm64__ */
278
279 #if __arm64__
280 extern int fourk_pager_data_request_debug;
281 SYSCTL_INT(_vm, OID_AUTO, fourk_pager_data_request_debug, CTLFLAG_RW | CTLFLAG_LOCKED, &fourk_pager_data_request_debug, 0, "");
282 #endif /* __arm64__ */
283 #endif /* DEVELOPMENT || DEBUG */
284
285 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_compressor, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_compressor, 0, "");
286 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_compressor_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_compressor_pages, 0, "");
287 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_terminate, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_terminate, 0, "");
288 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_terminate_failure, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_terminate_failure, 0, "");
289 SYSCTL_INT(_vm, OID_AUTO, vm_should_cow_but_wired, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.should_cow_but_wired, 0, "");
290 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_extra_cow, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_extra_cow, 0, "");
291 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_extra_cow_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_extra_cow_pages, 0, "");
292 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_lookup_failure_write, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_lookup_failure_write, 0, "");
293 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_lookup_failure_copy, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_lookup_failure_copy, 0, "");
294 #if VM_SCAN_FOR_SHADOW_CHAIN
295 static int vm_shadow_max_enabled = 0; /* Disabled by default */
296 extern int proc_shadow_max(void);
297 static int
298 vm_shadow_max SYSCTL_HANDLER_ARGS
299 {
300 #pragma unused(arg1, arg2, oidp)
301 int value = 0;
302
303 if (vm_shadow_max_enabled) {
304 value = proc_shadow_max();
305 }
306
307 return SYSCTL_OUT(req, &value, sizeof(value));
308 }
309 SYSCTL_PROC(_vm, OID_AUTO, vm_shadow_max, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
310 0, 0, &vm_shadow_max, "I", "");
311
312 SYSCTL_INT(_vm, OID_AUTO, vm_shadow_max_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_shadow_max_enabled, 0, "");
313
314 #endif /* VM_SCAN_FOR_SHADOW_CHAIN */
315
316 SYSCTL_INT(_vm, OID_AUTO, vm_debug_events, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_debug_events, 0, "");
317
318 __attribute__((noinline)) int __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(
319 mach_port_t task_access_port, int32_t calling_pid, uint32_t calling_gid, int32_t target_pid);
320 /*
321 * Sysctl's related to data/stack execution. See osfmk/vm/vm_map.c
322 */
323
324 #if DEVELOPMENT || DEBUG
325 extern int allow_stack_exec, allow_data_exec;
326
327 SYSCTL_INT(_vm, OID_AUTO, allow_stack_exec, CTLFLAG_RW | CTLFLAG_LOCKED, &allow_stack_exec, 0, "");
328 SYSCTL_INT(_vm, OID_AUTO, allow_data_exec, CTLFLAG_RW | CTLFLAG_LOCKED, &allow_data_exec, 0, "");
329
330 #endif /* DEVELOPMENT || DEBUG */
331
332 static const char *prot_values[] = {
333 "none",
334 "read-only",
335 "write-only",
336 "read-write",
337 "execute-only",
338 "read-execute",
339 "write-execute",
340 "read-write-execute"
341 };
342
343 void
344 log_stack_execution_failure(addr64_t vaddr, vm_prot_t prot)
345 {
346 printf("Data/Stack execution not permitted: %s[pid %d] at virtual address 0x%qx, protections were %s\n",
347 current_proc()->p_comm, current_proc()->p_pid, vaddr, prot_values[prot & VM_PROT_ALL]);
348 }
349
350 /*
351 * shared_region_unnest_logging: level of logging of unnesting events
352 * 0 - no logging
353 * 1 - throttled logging of unexpected unnesting events (default)
354 * 2 - unthrottled logging of unexpected unnesting events
355 * 3+ - unthrottled logging of all unnesting events
356 */
357 int shared_region_unnest_logging = 1;
358
359 SYSCTL_INT(_vm, OID_AUTO, shared_region_unnest_logging, CTLFLAG_RW | CTLFLAG_LOCKED,
360 &shared_region_unnest_logging, 0, "");
361
362 int vm_shared_region_unnest_log_interval = 10;
363 int shared_region_unnest_log_count_threshold = 5;
364
365 /*
366 * Shared cache path enforcement.
367 */
368
369 #if XNU_TARGET_OS_OSX
370
371 #if defined (__x86_64__)
372 static int scdir_enforce = 1;
373 #else /* defined (__x86_64__) */
374 static int scdir_enforce = 0; /* AOT caches live elsewhere */
375 #endif /* defined (__x86_64__) */
376
377 static char scdir_path[] = "/System/Library/dyld/";
378
379 #else /* XNU_TARGET_OS_OSX */
380
381 static int scdir_enforce = 0;
382 static char scdir_path[] = "/System/Library/Caches/com.apple.dyld/";
383
384 #endif /* XNU_TARGET_OS_OSX */
385
386 #ifndef SECURE_KERNEL
387 static int sysctl_scdir_enforce SYSCTL_HANDLER_ARGS
388 {
389 #if CONFIG_CSR
390 if (csr_check(CSR_ALLOW_UNRESTRICTED_FS) != 0) {
391 printf("Failed attempt to set vm.enforce_shared_cache_dir sysctl\n");
392 return EPERM;
393 }
394 #endif /* CONFIG_CSR */
395 return sysctl_handle_int(oidp, arg1, arg2, req);
396 }
397
398 SYSCTL_PROC(_vm, OID_AUTO, enforce_shared_cache_dir, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &scdir_enforce, 0, sysctl_scdir_enforce, "I", "");
399 #endif
400
401 /* These log rate throttling state variables aren't thread safe, but
402 * are sufficient unto the task.
403 */
404 static int64_t last_unnest_log_time = 0;
405 static int shared_region_unnest_log_count = 0;
406
407 void
408 log_unnest_badness(
409 vm_map_t m,
410 vm_map_offset_t s,
411 vm_map_offset_t e,
412 boolean_t is_nested_map,
413 vm_map_offset_t lowest_unnestable_addr)
414 {
415 struct timeval tv;
416
417 if (shared_region_unnest_logging == 0) {
418 return;
419 }
420
421 if (shared_region_unnest_logging <= 2 &&
422 is_nested_map &&
423 s >= lowest_unnestable_addr) {
424 /*
425 * Unnesting of writable map entries is fine.
426 */
427 return;
428 }
429
430 if (shared_region_unnest_logging <= 1) {
431 microtime(&tv);
432 if ((tv.tv_sec - last_unnest_log_time) <
433 vm_shared_region_unnest_log_interval) {
434 if (shared_region_unnest_log_count++ >
435 shared_region_unnest_log_count_threshold) {
436 return;
437 }
438 } else {
439 last_unnest_log_time = tv.tv_sec;
440 shared_region_unnest_log_count = 0;
441 }
442 }
443
444 DTRACE_VM4(log_unnest_badness,
445 vm_map_t, m,
446 vm_map_offset_t, s,
447 vm_map_offset_t, e,
448 vm_map_offset_t, lowest_unnestable_addr);
449 printf("%s[%d] triggered unnest of range 0x%qx->0x%qx of DYLD shared region in VM map %p. While not abnormal for debuggers, this increases system memory footprint until the target exits.\n", current_proc()->p_comm, current_proc()->p_pid, (uint64_t)s, (uint64_t)e, (void *) VM_KERNEL_ADDRPERM(m));
450 }
451
452 int
453 useracc(
454 user_addr_t addr,
455 user_size_t len,
456 int prot)
457 {
458 vm_map_t map;
459
460 map = current_map();
461 return vm_map_check_protection(
462 map,
463 vm_map_trunc_page(addr,
464 vm_map_page_mask(map)),
465 vm_map_round_page(addr + len,
466 vm_map_page_mask(map)),
467 prot == B_READ ? VM_PROT_READ : VM_PROT_WRITE);
468 }
469
470 int
471 vslock(
472 user_addr_t addr,
473 user_size_t len)
474 {
475 kern_return_t kret;
476 vm_map_t map;
477
478 map = current_map();
479 kret = vm_map_wire_kernel(map,
480 vm_map_trunc_page(addr,
481 vm_map_page_mask(map)),
482 vm_map_round_page(addr + len,
483 vm_map_page_mask(map)),
484 VM_PROT_READ | VM_PROT_WRITE, VM_KERN_MEMORY_BSD,
485 FALSE);
486
487 switch (kret) {
488 case KERN_SUCCESS:
489 return 0;
490 case KERN_INVALID_ADDRESS:
491 case KERN_NO_SPACE:
492 return ENOMEM;
493 case KERN_PROTECTION_FAILURE:
494 return EACCES;
495 default:
496 return EINVAL;
497 }
498 }
499
500 int
501 vsunlock(
502 user_addr_t addr,
503 user_size_t len,
504 __unused int dirtied)
505 {
506 #if FIXME /* [ */
507 pmap_t pmap;
508 vm_page_t pg;
509 vm_map_offset_t vaddr;
510 ppnum_t paddr;
511 #endif /* FIXME ] */
512 kern_return_t kret;
513 vm_map_t map;
514
515 map = current_map();
516
517 #if FIXME /* [ */
518 if (dirtied) {
519 pmap = get_task_pmap(current_task());
520 for (vaddr = vm_map_trunc_page(addr, PAGE_MASK);
521 vaddr < vm_map_round_page(addr + len, PAGE_MASK);
522 vaddr += PAGE_SIZE) {
523 paddr = pmap_find_phys(pmap, vaddr);
524 pg = PHYS_TO_VM_PAGE(paddr);
525 vm_page_set_modified(pg);
526 }
527 }
528 #endif /* FIXME ] */
529 #ifdef lint
530 dirtied++;
531 #endif /* lint */
532 kret = vm_map_unwire(map,
533 vm_map_trunc_page(addr,
534 vm_map_page_mask(map)),
535 vm_map_round_page(addr + len,
536 vm_map_page_mask(map)),
537 FALSE);
538 switch (kret) {
539 case KERN_SUCCESS:
540 return 0;
541 case KERN_INVALID_ADDRESS:
542 case KERN_NO_SPACE:
543 return ENOMEM;
544 case KERN_PROTECTION_FAILURE:
545 return EACCES;
546 default:
547 return EINVAL;
548 }
549 }
550
551 int
552 subyte(
553 user_addr_t addr,
554 int byte)
555 {
556 char character;
557
558 character = (char)byte;
559 return copyout((void *)&(character), addr, sizeof(char)) == 0 ? 0 : -1;
560 }
561
562 int
563 suibyte(
564 user_addr_t addr,
565 int byte)
566 {
567 char character;
568
569 character = (char)byte;
570 return copyout((void *)&(character), addr, sizeof(char)) == 0 ? 0 : -1;
571 }
572
573 int
574 fubyte(user_addr_t addr)
575 {
576 unsigned char byte;
577
578 if (copyin(addr, (void *) &byte, sizeof(char))) {
579 return -1;
580 }
581 return byte;
582 }
583
584 int
585 fuibyte(user_addr_t addr)
586 {
587 unsigned char byte;
588
589 if (copyin(addr, (void *) &(byte), sizeof(char))) {
590 return -1;
591 }
592 return byte;
593 }
594
595 int
596 suword(
597 user_addr_t addr,
598 long word)
599 {
600 return copyout((void *) &word, addr, sizeof(int)) == 0 ? 0 : -1;
601 }
602
603 long
604 fuword(user_addr_t addr)
605 {
606 long word = 0;
607
608 if (copyin(addr, (void *) &word, sizeof(int))) {
609 return -1;
610 }
611 return word;
612 }
613
614 /* suiword and fuiword are the same as suword and fuword, respectively */
615
616 int
617 suiword(
618 user_addr_t addr,
619 long word)
620 {
621 return copyout((void *) &word, addr, sizeof(int)) == 0 ? 0 : -1;
622 }
623
624 long
625 fuiword(user_addr_t addr)
626 {
627 long word = 0;
628
629 if (copyin(addr, (void *) &word, sizeof(int))) {
630 return -1;
631 }
632 return word;
633 }
634
635 /*
636 * With a 32-bit kernel and mixed 32/64-bit user tasks, this interface allows the
637 * fetching and setting of process-sized size_t and pointer values.
638 */
639 int
640 sulong(user_addr_t addr, int64_t word)
641 {
642 if (IS_64BIT_PROCESS(current_proc())) {
643 return copyout((void *)&word, addr, sizeof(word)) == 0 ? 0 : -1;
644 } else {
645 return suiword(addr, (long)word);
646 }
647 }
648
649 int64_t
650 fulong(user_addr_t addr)
651 {
652 int64_t longword;
653
654 if (IS_64BIT_PROCESS(current_proc())) {
655 if (copyin(addr, (void *)&longword, sizeof(longword)) != 0) {
656 return -1;
657 }
658 return longword;
659 } else {
660 return (int64_t)fuiword(addr);
661 }
662 }
663
664 int
665 suulong(user_addr_t addr, uint64_t uword)
666 {
667 if (IS_64BIT_PROCESS(current_proc())) {
668 return copyout((void *)&uword, addr, sizeof(uword)) == 0 ? 0 : -1;
669 } else {
670 return suiword(addr, (uint32_t)uword);
671 }
672 }
673
674 uint64_t
675 fuulong(user_addr_t addr)
676 {
677 uint64_t ulongword;
678
679 if (IS_64BIT_PROCESS(current_proc())) {
680 if (copyin(addr, (void *)&ulongword, sizeof(ulongword)) != 0) {
681 return -1ULL;
682 }
683 return ulongword;
684 } else {
685 return (uint64_t)fuiword(addr);
686 }
687 }
688
689 int
690 swapon(__unused proc_t procp, __unused struct swapon_args *uap, __unused int *retval)
691 {
692 return ENOTSUP;
693 }
694
695 /*
696 * pid_for_task
697 *
698 * Find the BSD process ID for the Mach task associated with the given Mach port
699 * name
700 *
701 * Parameters: args User argument descriptor (see below)
702 *
703 * Indirect parameters: args->t Mach port name
704 * args->pid Process ID (returned value; see below)
705 *
706 * Returns: KERL_SUCCESS Success
707 * KERN_FAILURE Not success
708 *
709 * Implicit returns: args->pid Process ID
710 *
711 */
712 kern_return_t
713 pid_for_task(
714 struct pid_for_task_args *args)
715 {
716 mach_port_name_t t = args->t;
717 user_addr_t pid_addr = args->pid;
718 proc_t p;
719 task_t t1;
720 int pid = -1;
721 kern_return_t err = KERN_SUCCESS;
722
723 AUDIT_MACH_SYSCALL_ENTER(AUE_PIDFORTASK);
724 AUDIT_ARG(mach_port1, t);
725
726 t1 = port_name_to_task_name(t);
727
728 if (t1 == TASK_NULL) {
729 err = KERN_FAILURE;
730 goto pftout;
731 } else {
732 p = get_bsdtask_info(t1);
733 if (p) {
734 pid = proc_pid(p);
735 err = KERN_SUCCESS;
736 } else if (is_corpsetask(t1)) {
737 pid = task_pid(t1);
738 err = KERN_SUCCESS;
739 } else {
740 err = KERN_FAILURE;
741 }
742 }
743 task_deallocate(t1);
744 pftout:
745 AUDIT_ARG(pid, pid);
746 (void) copyout((char *) &pid, pid_addr, sizeof(int));
747 AUDIT_MACH_SYSCALL_EXIT(err);
748 return err;
749 }
750
751 /*
752 *
753 * tfp_policy = KERN_TFP_POLICY_DENY; Deny Mode: None allowed except for self
754 * tfp_policy = KERN_TFP_POLICY_DEFAULT; default mode: all posix checks and upcall via task port for authentication
755 *
756 */
757 static int tfp_policy = KERN_TFP_POLICY_DEFAULT;
758
759 /*
760 * Routine: task_for_pid_posix_check
761 * Purpose:
762 * Verify that the current process should be allowed to
763 * get the target process's task port. This is only
764 * permitted if:
765 * - The current process is root
766 * OR all of the following are true:
767 * - The target process's real, effective, and saved uids
768 * are the same as the current proc's euid,
769 * - The target process's group set is a subset of the
770 * calling process's group set, and
771 * - The target process hasn't switched credentials.
772 *
773 * Returns: TRUE: permitted
774 * FALSE: denied
775 */
776 static int
777 task_for_pid_posix_check(proc_t target)
778 {
779 kauth_cred_t targetcred, mycred;
780 uid_t myuid;
781 int allowed;
782
783 /* No task_for_pid on bad targets */
784 if (target->p_stat == SZOMB) {
785 return FALSE;
786 }
787
788 mycred = kauth_cred_get();
789 myuid = kauth_cred_getuid(mycred);
790
791 /* If we're running as root, the check passes */
792 if (kauth_cred_issuser(mycred)) {
793 return TRUE;
794 }
795
796 /* We're allowed to get our own task port */
797 if (target == current_proc()) {
798 return TRUE;
799 }
800
801 /*
802 * Under DENY, only root can get another proc's task port,
803 * so no more checks are needed.
804 */
805 if (tfp_policy == KERN_TFP_POLICY_DENY) {
806 return FALSE;
807 }
808
809 targetcred = kauth_cred_proc_ref(target);
810 allowed = TRUE;
811
812 /* Do target's ruid, euid, and saved uid match my euid? */
813 if ((kauth_cred_getuid(targetcred) != myuid) ||
814 (kauth_cred_getruid(targetcred) != myuid) ||
815 (kauth_cred_getsvuid(targetcred) != myuid)) {
816 allowed = FALSE;
817 goto out;
818 }
819
820 /* Are target's groups a subset of my groups? */
821 if (kauth_cred_gid_subset(targetcred, mycred, &allowed) ||
822 allowed == 0) {
823 allowed = FALSE;
824 goto out;
825 }
826
827 /* Has target switched credentials? */
828 if (target->p_flag & P_SUGID) {
829 allowed = FALSE;
830 goto out;
831 }
832
833 out:
834 kauth_cred_unref(&targetcred);
835 return allowed;
836 }
837
838 /*
839 * __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__
840 *
841 * Description: Waits for the user space daemon to respond to the request
842 * we made. Function declared non inline to be visible in
843 * stackshots and spindumps as well as debugging.
844 */
845 __attribute__((noinline)) int
846 __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(
847 mach_port_t task_access_port, int32_t calling_pid, uint32_t calling_gid, int32_t target_pid)
848 {
849 return check_task_access(task_access_port, calling_pid, calling_gid, target_pid);
850 }
851
852 /*
853 * Routine: task_for_pid
854 * Purpose:
855 * Get the task port for another "process", named by its
856 * process ID on the same host as "target_task".
857 *
858 * Only permitted to privileged processes, or processes
859 * with the same user ID.
860 *
861 * Note: if pid == 0, an error is return no matter who is calling.
862 *
863 * XXX This should be a BSD system call, not a Mach trap!!!
864 */
865 kern_return_t
866 task_for_pid(
867 struct task_for_pid_args *args)
868 {
869 mach_port_name_t target_tport = args->target_tport;
870 int pid = args->pid;
871 user_addr_t task_addr = args->t;
872 proc_t p = PROC_NULL;
873 task_t t1 = TASK_NULL;
874 task_t task = TASK_NULL;
875 mach_port_name_t tret = MACH_PORT_NULL;
876 ipc_port_t tfpport = MACH_PORT_NULL;
877 void * sright = NULL;
878 int error = 0;
879 boolean_t is_current_proc = FALSE;
880 struct proc_ident pident = {0};
881
882 AUDIT_MACH_SYSCALL_ENTER(AUE_TASKFORPID);
883 AUDIT_ARG(pid, pid);
884 AUDIT_ARG(mach_port1, target_tport);
885
886 /* Always check if pid == 0 */
887 if (pid == 0) {
888 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
889 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
890 return KERN_FAILURE;
891 }
892
893 t1 = port_name_to_task(target_tport);
894 if (t1 == TASK_NULL) {
895 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
896 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
897 return KERN_FAILURE;
898 }
899
900
901 p = proc_find(pid);
902 if (p == PROC_NULL) {
903 error = KERN_FAILURE;
904 goto tfpout;
905 }
906 pident = proc_ident(p);
907 is_current_proc = (p == current_proc());
908
909 #if CONFIG_AUDIT
910 AUDIT_ARG(process, p);
911 #endif
912
913 if (!(task_for_pid_posix_check(p))) {
914 error = KERN_FAILURE;
915 goto tfpout;
916 }
917
918 if (p->task == TASK_NULL) {
919 error = KERN_SUCCESS;
920 goto tfpout;
921 }
922
923 /*
924 * Grab a task reference and drop the proc reference as the proc ref
925 * shouldn't be held accross upcalls.
926 */
927 task = p->task;
928 task_reference(task);
929
930 proc_rele(p);
931 p = PROC_NULL;
932
933 #if CONFIG_MACF
934 error = mac_proc_check_get_task(kauth_cred_get(), &pident);
935 if (error) {
936 error = KERN_FAILURE;
937 goto tfpout;
938 }
939 #endif
940
941 /* If we aren't root and target's task access port is set... */
942 if (!kauth_cred_issuser(kauth_cred_get()) &&
943 !is_current_proc &&
944 (task_get_task_access_port(task, &tfpport) == 0) &&
945 (tfpport != IPC_PORT_NULL)) {
946 if (tfpport == IPC_PORT_DEAD) {
947 error = KERN_PROTECTION_FAILURE;
948 goto tfpout;
949 }
950
951 /* Call up to the task access server */
952 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
953
954 if (error != MACH_MSG_SUCCESS) {
955 if (error == MACH_RCV_INTERRUPTED) {
956 error = KERN_ABORTED;
957 } else {
958 error = KERN_FAILURE;
959 }
960 goto tfpout;
961 }
962 }
963
964 /* Grant task port access */
965 extmod_statistics_incr_task_for_pid(task);
966 sright = (void *) convert_task_to_port(task);
967
968 /* Check if the task has been corpsified */
969 if (is_corpsetask(task)) {
970 /* task ref consumed by convert_task_to_port */
971 task = TASK_NULL;
972 ipc_port_release_send(sright);
973 error = KERN_FAILURE;
974 goto tfpout;
975 }
976
977 /* task ref consumed by convert_task_to_port */
978 task = TASK_NULL;
979 tret = ipc_port_copyout_send(
980 sright,
981 get_task_ipcspace(current_task()));
982
983 error = KERN_SUCCESS;
984
985 tfpout:
986 task_deallocate(t1);
987 AUDIT_ARG(mach_port2, tret);
988 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
989
990 if (tfpport != IPC_PORT_NULL) {
991 ipc_port_release_send(tfpport);
992 }
993 if (task != TASK_NULL) {
994 task_deallocate(task);
995 }
996 if (p != PROC_NULL) {
997 proc_rele(p);
998 }
999 AUDIT_MACH_SYSCALL_EXIT(error);
1000 return error;
1001 }
1002
1003 /*
1004 * Routine: task_name_for_pid
1005 * Purpose:
1006 * Get the task name port for another "process", named by its
1007 * process ID on the same host as "target_task".
1008 *
1009 * Only permitted to privileged processes, or processes
1010 * with the same user ID.
1011 *
1012 * XXX This should be a BSD system call, not a Mach trap!!!
1013 */
1014
1015 kern_return_t
1016 task_name_for_pid(
1017 struct task_name_for_pid_args *args)
1018 {
1019 mach_port_name_t target_tport = args->target_tport;
1020 int pid = args->pid;
1021 user_addr_t task_addr = args->t;
1022 proc_t p = PROC_NULL;
1023 task_t t1;
1024 mach_port_name_t tret;
1025 void * sright;
1026 int error = 0, refheld = 0;
1027 kauth_cred_t target_cred;
1028
1029 AUDIT_MACH_SYSCALL_ENTER(AUE_TASKNAMEFORPID);
1030 AUDIT_ARG(pid, pid);
1031 AUDIT_ARG(mach_port1, target_tport);
1032
1033 t1 = port_name_to_task(target_tport);
1034 if (t1 == TASK_NULL) {
1035 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
1036 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
1037 return KERN_FAILURE;
1038 }
1039
1040 p = proc_find(pid);
1041 if (p != PROC_NULL) {
1042 AUDIT_ARG(process, p);
1043 target_cred = kauth_cred_proc_ref(p);
1044 refheld = 1;
1045
1046 if ((p->p_stat != SZOMB)
1047 && ((current_proc() == p)
1048 || kauth_cred_issuser(kauth_cred_get())
1049 || ((kauth_cred_getuid(target_cred) == kauth_cred_getuid(kauth_cred_get())) &&
1050 ((kauth_cred_getruid(target_cred) == kauth_getruid()))))) {
1051 if (p->task != TASK_NULL) {
1052 struct proc_ident pident = proc_ident(p);
1053
1054 task_t task = p->task;
1055
1056 task_reference(p->task);
1057 proc_rele(p);
1058 p = PROC_NULL;
1059 #if CONFIG_MACF
1060 error = mac_proc_check_get_task_name(kauth_cred_get(), &pident);
1061 if (error) {
1062 task_deallocate(task);
1063 goto noperm;
1064 }
1065 #endif
1066 sright = (void *)convert_task_name_to_port(task);
1067 task = NULL;
1068 tret = ipc_port_copyout_send(sright,
1069 get_task_ipcspace(current_task()));
1070 } else {
1071 tret = MACH_PORT_NULL;
1072 }
1073
1074 AUDIT_ARG(mach_port2, tret);
1075 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
1076 task_deallocate(t1);
1077 error = KERN_SUCCESS;
1078 goto tnfpout;
1079 }
1080 }
1081
1082 #if CONFIG_MACF
1083 noperm:
1084 #endif
1085 task_deallocate(t1);
1086 tret = MACH_PORT_NULL;
1087 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1088 error = KERN_FAILURE;
1089 tnfpout:
1090 if (refheld != 0) {
1091 kauth_cred_unref(&target_cred);
1092 }
1093 if (p != PROC_NULL) {
1094 proc_rele(p);
1095 }
1096 AUDIT_MACH_SYSCALL_EXIT(error);
1097 return error;
1098 }
1099
1100 /*
1101 * Routine: task_inspect_for_pid
1102 * Purpose:
1103 * Get the task inspect port for another "process", named by its
1104 * process ID on the same host as "target_task".
1105 */
1106 int
1107 task_inspect_for_pid(struct proc *p __unused, struct task_inspect_for_pid_args *args, int *ret)
1108 {
1109 mach_port_name_t target_tport = args->target_tport;
1110 int pid = args->pid;
1111 user_addr_t task_addr = args->t;
1112
1113 proc_t proc = PROC_NULL;
1114 task_t t1 = TASK_NULL;
1115 task_inspect_t task_insp = TASK_INSPECT_NULL;
1116 mach_port_name_t tret = MACH_PORT_NULL;
1117 ipc_port_t tfpport = MACH_PORT_NULL;
1118 int error = 0;
1119 void *sright = NULL;
1120 boolean_t is_current_proc = FALSE;
1121 struct proc_ident pident = {0};
1122
1123 /* Disallow inspect port for kernel_task */
1124 if (pid == 0) {
1125 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
1126 return EPERM;
1127 }
1128
1129 t1 = port_name_to_task(target_tport);
1130 if (t1 == TASK_NULL) {
1131 (void) copyout((char *) &t1, task_addr, sizeof(mach_port_name_t));
1132 return EINVAL;
1133 }
1134
1135 proc = proc_find(pid);
1136 if (proc == PROC_NULL) {
1137 error = ESRCH;
1138 goto tifpout;
1139 }
1140 pident = proc_ident(proc);
1141 is_current_proc = (proc == current_proc());
1142
1143 if (!(task_for_pid_posix_check(proc))) {
1144 error = EPERM;
1145 goto tifpout;
1146 }
1147
1148 task_insp = proc->task;
1149 if (task_insp == TASK_INSPECT_NULL) {
1150 goto tifpout;
1151 }
1152
1153 /*
1154 * Grab a task reference and drop the proc reference before making any upcalls.
1155 */
1156 task_reference(task_insp);
1157
1158 proc_rele(proc);
1159 proc = PROC_NULL;
1160
1161 /*
1162 * For now, it performs the same set of permission checks as task_for_pid. This
1163 * will be addressed in rdar://problem/53478660
1164 */
1165 #if CONFIG_MACF
1166 error = mac_proc_check_get_task(kauth_cred_get(), &pident);
1167 if (error) {
1168 error = EPERM;
1169 goto tifpout;
1170 }
1171 #endif
1172
1173 /* If we aren't root and target's task access port is set... */
1174 if (!kauth_cred_issuser(kauth_cred_get()) &&
1175 !is_current_proc &&
1176 (task_get_task_access_port(task_insp, &tfpport) == 0) &&
1177 (tfpport != IPC_PORT_NULL)) {
1178 if (tfpport == IPC_PORT_DEAD) {
1179 error = EACCES;
1180 goto tifpout;
1181 }
1182
1183
1184 /* Call up to the task access server */
1185 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
1186
1187 if (error != MACH_MSG_SUCCESS) {
1188 if (error == MACH_RCV_INTERRUPTED) {
1189 error = EINTR;
1190 } else {
1191 error = EPERM;
1192 }
1193 goto tifpout;
1194 }
1195 }
1196
1197 /* Check if the task has been corpsified */
1198 if (is_corpsetask(task_insp)) {
1199 error = EACCES;
1200 goto tifpout;
1201 }
1202
1203 /* could be IP_NULL, consumes a ref */
1204 sright = (void*) convert_task_inspect_to_port(task_insp);
1205 task_insp = TASK_INSPECT_NULL;
1206 tret = ipc_port_copyout_send(sright, get_task_ipcspace(current_task()));
1207
1208 tifpout:
1209 task_deallocate(t1);
1210 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1211 if (proc != PROC_NULL) {
1212 proc_rele(proc);
1213 }
1214 if (tfpport != IPC_PORT_NULL) {
1215 ipc_port_release_send(tfpport);
1216 }
1217 if (task_insp != TASK_INSPECT_NULL) {
1218 task_deallocate(task_insp);
1219 }
1220
1221 *ret = error;
1222 return error;
1223 }
1224
1225 /*
1226 * Routine: task_read_for_pid
1227 * Purpose:
1228 * Get the task read port for another "process", named by its
1229 * process ID on the same host as "target_task".
1230 */
1231 int
1232 task_read_for_pid(struct proc *p __unused, struct task_read_for_pid_args *args, int *ret)
1233 {
1234 mach_port_name_t target_tport = args->target_tport;
1235 int pid = args->pid;
1236 user_addr_t task_addr = args->t;
1237
1238 proc_t proc = PROC_NULL;
1239 task_t t1 = TASK_NULL;
1240 task_read_t task_read = TASK_READ_NULL;
1241 mach_port_name_t tret = MACH_PORT_NULL;
1242 ipc_port_t tfpport = MACH_PORT_NULL;
1243 int error = 0;
1244 void *sright = NULL;
1245 boolean_t is_current_proc = FALSE;
1246 struct proc_ident pident = {0};
1247
1248 /* Disallow read port for kernel_task */
1249 if (pid == 0) {
1250 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
1251 return EPERM;
1252 }
1253
1254 t1 = port_name_to_task(target_tport);
1255 if (t1 == TASK_NULL) {
1256 (void) copyout((char *) &t1, task_addr, sizeof(mach_port_name_t));
1257 return EINVAL;
1258 }
1259
1260 proc = proc_find(pid);
1261 if (proc == PROC_NULL) {
1262 error = ESRCH;
1263 goto trfpout;
1264 }
1265 pident = proc_ident(proc);
1266 is_current_proc = (proc == current_proc());
1267
1268 if (!(task_for_pid_posix_check(proc))) {
1269 error = EPERM;
1270 goto trfpout;
1271 }
1272
1273 task_read = proc->task;
1274 if (task_read == TASK_INSPECT_NULL) {
1275 goto trfpout;
1276 }
1277
1278 /*
1279 * Grab a task reference and drop the proc reference before making any upcalls.
1280 */
1281 task_reference(task_read);
1282
1283 proc_rele(proc);
1284 proc = PROC_NULL;
1285
1286 /*
1287 * For now, it performs the same set of permission checks as task_for_pid. This
1288 * will be addressed in rdar://problem/53478660
1289 */
1290 #if CONFIG_MACF
1291 error = mac_proc_check_get_task(kauth_cred_get(), &pident);
1292 if (error) {
1293 error = EPERM;
1294 goto trfpout;
1295 }
1296 #endif
1297
1298 /* If we aren't root and target's task access port is set... */
1299 if (!kauth_cred_issuser(kauth_cred_get()) &&
1300 !is_current_proc &&
1301 (task_get_task_access_port(task_read, &tfpport) == 0) &&
1302 (tfpport != IPC_PORT_NULL)) {
1303 if (tfpport == IPC_PORT_DEAD) {
1304 error = EACCES;
1305 goto trfpout;
1306 }
1307
1308
1309 /* Call up to the task access server */
1310 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
1311
1312 if (error != MACH_MSG_SUCCESS) {
1313 if (error == MACH_RCV_INTERRUPTED) {
1314 error = EINTR;
1315 } else {
1316 error = EPERM;
1317 }
1318 goto trfpout;
1319 }
1320 }
1321
1322 /* Check if the task has been corpsified */
1323 if (is_corpsetask(task_read)) {
1324 error = EACCES;
1325 goto trfpout;
1326 }
1327
1328 /* could be IP_NULL, consumes a ref */
1329 sright = (void*) convert_task_read_to_port(task_read);
1330 task_read = TASK_READ_NULL;
1331 tret = ipc_port_copyout_send(sright, get_task_ipcspace(current_task()));
1332
1333 trfpout:
1334 task_deallocate(t1);
1335 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1336 if (proc != PROC_NULL) {
1337 proc_rele(proc);
1338 }
1339 if (tfpport != IPC_PORT_NULL) {
1340 ipc_port_release_send(tfpport);
1341 }
1342 if (task_read != TASK_READ_NULL) {
1343 task_deallocate(task_read);
1344 }
1345
1346 *ret = error;
1347 return error;
1348 }
1349
1350 kern_return_t
1351 pid_suspend(struct proc *p __unused, struct pid_suspend_args *args, int *ret)
1352 {
1353 task_t target = NULL;
1354 proc_t targetproc = PROC_NULL;
1355 int pid = args->pid;
1356 int error = 0;
1357 mach_port_t tfpport = MACH_PORT_NULL;
1358
1359 if (pid == 0) {
1360 error = EPERM;
1361 goto out;
1362 }
1363
1364 targetproc = proc_find(pid);
1365 if (targetproc == PROC_NULL) {
1366 error = ESRCH;
1367 goto out;
1368 }
1369
1370 if (!task_for_pid_posix_check(targetproc) &&
1371 !IOTaskHasEntitlement(current_task(), PROCESS_RESUME_SUSPEND_ENTITLEMENT)) {
1372 error = EPERM;
1373 goto out;
1374 }
1375
1376 #if CONFIG_MACF
1377 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_SUSPEND);
1378 if (error) {
1379 error = EPERM;
1380 goto out;
1381 }
1382 #endif
1383
1384 target = targetproc->task;
1385 #ifndef CONFIG_EMBEDDED
1386 if (target != TASK_NULL) {
1387 /* If we aren't root and target's task access port is set... */
1388 if (!kauth_cred_issuser(kauth_cred_get()) &&
1389 targetproc != current_proc() &&
1390 (task_get_task_access_port(target, &tfpport) == 0) &&
1391 (tfpport != IPC_PORT_NULL)) {
1392 if (tfpport == IPC_PORT_DEAD) {
1393 error = EACCES;
1394 goto out;
1395 }
1396
1397 /* Call up to the task access server */
1398 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
1399
1400 if (error != MACH_MSG_SUCCESS) {
1401 if (error == MACH_RCV_INTERRUPTED) {
1402 error = EINTR;
1403 } else {
1404 error = EPERM;
1405 }
1406 goto out;
1407 }
1408 }
1409 }
1410 #endif
1411
1412 task_reference(target);
1413 error = task_pidsuspend(target);
1414 if (error) {
1415 if (error == KERN_INVALID_ARGUMENT) {
1416 error = EINVAL;
1417 } else {
1418 error = EPERM;
1419 }
1420 }
1421 #if CONFIG_MEMORYSTATUS
1422 else {
1423 memorystatus_on_suspend(targetproc);
1424 }
1425 #endif
1426
1427 task_deallocate(target);
1428
1429 out:
1430 if (tfpport != IPC_PORT_NULL) {
1431 ipc_port_release_send(tfpport);
1432 }
1433
1434 if (targetproc != PROC_NULL) {
1435 proc_rele(targetproc);
1436 }
1437 *ret = error;
1438 return error;
1439 }
1440
1441 kern_return_t
1442 debug_control_port_for_pid(struct debug_control_port_for_pid_args *args)
1443 {
1444 mach_port_name_t target_tport = args->target_tport;
1445 int pid = args->pid;
1446 user_addr_t task_addr = args->t;
1447 proc_t p = PROC_NULL;
1448 task_t t1 = TASK_NULL;
1449 task_t task = TASK_NULL;
1450 mach_port_name_t tret = MACH_PORT_NULL;
1451 ipc_port_t tfpport = MACH_PORT_NULL;
1452 ipc_port_t sright = NULL;
1453 int error = 0;
1454 boolean_t is_current_proc = FALSE;
1455 struct proc_ident pident = {0};
1456
1457 AUDIT_MACH_SYSCALL_ENTER(AUE_DBGPORTFORPID);
1458 AUDIT_ARG(pid, pid);
1459 AUDIT_ARG(mach_port1, target_tport);
1460
1461 /* Always check if pid == 0 */
1462 if (pid == 0) {
1463 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
1464 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
1465 return KERN_FAILURE;
1466 }
1467
1468 t1 = port_name_to_task(target_tport);
1469 if (t1 == TASK_NULL) {
1470 (void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
1471 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
1472 return KERN_FAILURE;
1473 }
1474
1475 p = proc_find(pid);
1476 if (p == PROC_NULL) {
1477 error = KERN_FAILURE;
1478 goto tfpout;
1479 }
1480 pident = proc_ident(p);
1481 is_current_proc = (p == current_proc());
1482
1483 #if CONFIG_AUDIT
1484 AUDIT_ARG(process, p);
1485 #endif
1486
1487 if (!(task_for_pid_posix_check(p))) {
1488 error = KERN_FAILURE;
1489 goto tfpout;
1490 }
1491
1492 if (p->task == TASK_NULL) {
1493 error = KERN_SUCCESS;
1494 goto tfpout;
1495 }
1496
1497 /*
1498 * Grab a task reference and drop the proc reference before making any upcalls.
1499 */
1500 task = p->task;
1501 task_reference(task);
1502
1503 proc_rele(p);
1504 p = PROC_NULL;
1505
1506 if (!IOTaskHasEntitlement(current_task(), DEBUG_PORT_ENTITLEMENT)) {
1507 #if CONFIG_MACF
1508 error = mac_proc_check_get_task(kauth_cred_get(), &pident);
1509 if (error) {
1510 error = KERN_FAILURE;
1511 goto tfpout;
1512 }
1513 #endif
1514
1515 /* If we aren't root and target's task access port is set... */
1516 if (!kauth_cred_issuser(kauth_cred_get()) &&
1517 !is_current_proc &&
1518 (task_get_task_access_port(task, &tfpport) == 0) &&
1519 (tfpport != IPC_PORT_NULL)) {
1520 if (tfpport == IPC_PORT_DEAD) {
1521 error = KERN_PROTECTION_FAILURE;
1522 goto tfpout;
1523 }
1524
1525
1526 /* Call up to the task access server */
1527 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
1528
1529 if (error != MACH_MSG_SUCCESS) {
1530 if (error == MACH_RCV_INTERRUPTED) {
1531 error = KERN_ABORTED;
1532 } else {
1533 error = KERN_FAILURE;
1534 }
1535 goto tfpout;
1536 }
1537 }
1538 }
1539
1540 /* Check if the task has been corpsified */
1541 if (is_corpsetask(task)) {
1542 error = KERN_FAILURE;
1543 goto tfpout;
1544 }
1545
1546 error = task_get_debug_control_port(task, &sright);
1547 if (error != KERN_SUCCESS) {
1548 goto tfpout;
1549 }
1550
1551 tret = ipc_port_copyout_send(
1552 sright,
1553 get_task_ipcspace(current_task()));
1554
1555 error = KERN_SUCCESS;
1556
1557 tfpout:
1558 task_deallocate(t1);
1559 AUDIT_ARG(mach_port2, tret);
1560 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1561
1562 if (tfpport != IPC_PORT_NULL) {
1563 ipc_port_release_send(tfpport);
1564 }
1565 if (task != TASK_NULL) {
1566 task_deallocate(task);
1567 }
1568 if (p != PROC_NULL) {
1569 proc_rele(p);
1570 }
1571 AUDIT_MACH_SYSCALL_EXIT(error);
1572 return error;
1573 }
1574
1575 kern_return_t
1576 pid_resume(struct proc *p __unused, struct pid_resume_args *args, int *ret)
1577 {
1578 task_t target = NULL;
1579 proc_t targetproc = PROC_NULL;
1580 int pid = args->pid;
1581 int error = 0;
1582 mach_port_t tfpport = MACH_PORT_NULL;
1583
1584 if (pid == 0) {
1585 error = EPERM;
1586 goto out;
1587 }
1588
1589 targetproc = proc_find(pid);
1590 if (targetproc == PROC_NULL) {
1591 error = ESRCH;
1592 goto out;
1593 }
1594
1595 if (!task_for_pid_posix_check(targetproc) &&
1596 !IOTaskHasEntitlement(current_task(), PROCESS_RESUME_SUSPEND_ENTITLEMENT)) {
1597 error = EPERM;
1598 goto out;
1599 }
1600
1601 #if CONFIG_MACF
1602 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_RESUME);
1603 if (error) {
1604 error = EPERM;
1605 goto out;
1606 }
1607 #endif
1608
1609 target = targetproc->task;
1610 #ifndef CONFIG_EMBEDDED
1611 if (target != TASK_NULL) {
1612 /* If we aren't root and target's task access port is set... */
1613 if (!kauth_cred_issuser(kauth_cred_get()) &&
1614 targetproc != current_proc() &&
1615 (task_get_task_access_port(target, &tfpport) == 0) &&
1616 (tfpport != IPC_PORT_NULL)) {
1617 if (tfpport == IPC_PORT_DEAD) {
1618 error = EACCES;
1619 goto out;
1620 }
1621
1622 /* Call up to the task access server */
1623 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
1624
1625 if (error != MACH_MSG_SUCCESS) {
1626 if (error == MACH_RCV_INTERRUPTED) {
1627 error = EINTR;
1628 } else {
1629 error = EPERM;
1630 }
1631 goto out;
1632 }
1633 }
1634 }
1635 #endif
1636
1637 #if !XNU_TARGET_OS_OSX
1638 #if SOCKETS
1639 resume_proc_sockets(targetproc);
1640 #endif /* SOCKETS */
1641 #endif /* !XNU_TARGET_OS_OSX */
1642
1643 task_reference(target);
1644
1645 #if CONFIG_MEMORYSTATUS
1646 memorystatus_on_resume(targetproc);
1647 #endif
1648
1649 error = task_pidresume(target);
1650 if (error) {
1651 if (error == KERN_INVALID_ARGUMENT) {
1652 error = EINVAL;
1653 } else {
1654 if (error == KERN_MEMORY_ERROR) {
1655 psignal(targetproc, SIGKILL);
1656 error = EIO;
1657 } else {
1658 error = EPERM;
1659 }
1660 }
1661 }
1662
1663 task_deallocate(target);
1664
1665 out:
1666 if (tfpport != IPC_PORT_NULL) {
1667 ipc_port_release_send(tfpport);
1668 }
1669
1670 if (targetproc != PROC_NULL) {
1671 proc_rele(targetproc);
1672 }
1673
1674 *ret = error;
1675 return error;
1676 }
1677
1678 #if CONFIG_EMBEDDED
1679 /*
1680 * Freeze the specified process (provided in args->pid), or find and freeze a PID.
1681 * When a process is specified, this call is blocking, otherwise we wake up the
1682 * freezer thread and do not block on a process being frozen.
1683 */
1684 kern_return_t
1685 pid_hibernate(struct proc *p __unused, struct pid_hibernate_args *args, int *ret)
1686 {
1687 int error = 0;
1688 proc_t targetproc = PROC_NULL;
1689 int pid = args->pid;
1690
1691 #ifndef CONFIG_FREEZE
1692 #pragma unused(pid)
1693 #else
1694
1695 /*
1696 * If a pid has been provided, we obtain the process handle and call task_for_pid_posix_check().
1697 */
1698
1699 if (pid >= 0) {
1700 targetproc = proc_find(pid);
1701
1702 if (targetproc == PROC_NULL) {
1703 error = ESRCH;
1704 goto out;
1705 }
1706
1707 if (!task_for_pid_posix_check(targetproc)) {
1708 error = EPERM;
1709 goto out;
1710 }
1711 }
1712
1713 #if CONFIG_MACF
1714 //Note that targetproc may be null
1715 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_HIBERNATE);
1716 if (error) {
1717 error = EPERM;
1718 goto out;
1719 }
1720 #endif
1721
1722 if (pid == -2) {
1723 vm_pageout_anonymous_pages();
1724 } else if (pid == -1) {
1725 memorystatus_on_inactivity(targetproc);
1726 } else {
1727 error = memorystatus_freeze_process_sync(targetproc);
1728 }
1729
1730 out:
1731
1732 #endif /* CONFIG_FREEZE */
1733
1734 if (targetproc != PROC_NULL) {
1735 proc_rele(targetproc);
1736 }
1737 *ret = error;
1738 return error;
1739 }
1740 #endif /* CONFIG_EMBEDDED */
1741
1742 #if SOCKETS
1743 int
1744 networking_memstatus_callout(proc_t p, uint32_t status)
1745 {
1746 struct fileproc *fp;
1747
1748 /*
1749 * proc list lock NOT held
1750 * proc lock NOT held
1751 * a reference on the proc has been held / shall be dropped by the caller.
1752 */
1753 LCK_MTX_ASSERT(proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED);
1754 LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED);
1755
1756 proc_fdlock(p);
1757
1758 fdt_foreach(fp, p) {
1759 switch (FILEGLOB_DTYPE(fp->fp_glob)) {
1760 #if NECP
1761 case DTYPE_NETPOLICY:
1762 necp_fd_memstatus(p, status,
1763 (struct necp_fd_data *)fp->fp_glob->fg_data);
1764 break;
1765 #endif /* NECP */
1766 default:
1767 break;
1768 }
1769 }
1770 proc_fdunlock(p);
1771
1772 return 1;
1773 }
1774
1775
1776 static int
1777 networking_defunct_callout(proc_t p, void *arg)
1778 {
1779 struct pid_shutdown_sockets_args *args = arg;
1780 int pid = args->pid;
1781 int level = args->level;
1782 struct fileproc *fp;
1783
1784 proc_fdlock(p);
1785
1786 fdt_foreach(fp, p) {
1787 struct fileglob *fg = fp->fp_glob;
1788
1789 switch (FILEGLOB_DTYPE(fg)) {
1790 case DTYPE_SOCKET: {
1791 struct socket *so = (struct socket *)fg->fg_data;
1792 if (p->p_pid == pid || so->last_pid == pid ||
1793 ((so->so_flags & SOF_DELEGATED) && so->e_pid == pid)) {
1794 /* Call networking stack with socket and level */
1795 (void)socket_defunct(p, so, level);
1796 }
1797 break;
1798 }
1799 #if NECP
1800 case DTYPE_NETPOLICY:
1801 /* first pass: defunct necp and get stats for ntstat */
1802 if (p->p_pid == pid) {
1803 necp_fd_defunct(p,
1804 (struct necp_fd_data *)fg->fg_data);
1805 }
1806 break;
1807 #endif /* NECP */
1808 default:
1809 break;
1810 }
1811 }
1812
1813 proc_fdunlock(p);
1814
1815 return PROC_RETURNED;
1816 }
1817
1818 int
1819 pid_shutdown_sockets(struct proc *p __unused, struct pid_shutdown_sockets_args *args, int *ret)
1820 {
1821 int error = 0;
1822 proc_t targetproc = PROC_NULL;
1823 int pid = args->pid;
1824 int level = args->level;
1825
1826 if (level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_SVC &&
1827 level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL) {
1828 error = EINVAL;
1829 goto out;
1830 }
1831
1832 targetproc = proc_find(pid);
1833 if (targetproc == PROC_NULL) {
1834 error = ESRCH;
1835 goto out;
1836 }
1837
1838 if (!task_for_pid_posix_check(targetproc) &&
1839 !IOTaskHasEntitlement(current_task(), PROCESS_RESUME_SUSPEND_ENTITLEMENT)) {
1840 error = EPERM;
1841 goto out;
1842 }
1843
1844 #if CONFIG_MACF
1845 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_SHUTDOWN_SOCKETS);
1846 if (error) {
1847 error = EPERM;
1848 goto out;
1849 }
1850 #endif
1851
1852 proc_iterate(PROC_ALLPROCLIST | PROC_NOWAITTRANS,
1853 networking_defunct_callout, args, NULL, NULL);
1854
1855 out:
1856 if (targetproc != PROC_NULL) {
1857 proc_rele(targetproc);
1858 }
1859 *ret = error;
1860 return error;
1861 }
1862
1863 #endif /* SOCKETS */
1864
1865 static int
1866 sysctl_settfp_policy(__unused struct sysctl_oid *oidp, void *arg1,
1867 __unused int arg2, struct sysctl_req *req)
1868 {
1869 int error = 0;
1870 int new_value;
1871
1872 error = SYSCTL_OUT(req, arg1, sizeof(int));
1873 if (error || req->newptr == USER_ADDR_NULL) {
1874 return error;
1875 }
1876
1877 if (!kauth_cred_issuser(kauth_cred_get())) {
1878 return EPERM;
1879 }
1880
1881 if ((error = SYSCTL_IN(req, &new_value, sizeof(int)))) {
1882 goto out;
1883 }
1884 if ((new_value == KERN_TFP_POLICY_DENY)
1885 || (new_value == KERN_TFP_POLICY_DEFAULT)) {
1886 tfp_policy = new_value;
1887 } else {
1888 error = EINVAL;
1889 }
1890 out:
1891 return error;
1892 }
1893
1894 #if defined(SECURE_KERNEL)
1895 static int kern_secure_kernel = 1;
1896 #else
1897 static int kern_secure_kernel = 0;
1898 #endif
1899
1900 SYSCTL_INT(_kern, OID_AUTO, secure_kernel, CTLFLAG_RD | CTLFLAG_LOCKED, &kern_secure_kernel, 0, "");
1901
1902 SYSCTL_NODE(_kern, KERN_TFP, tfp, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "tfp");
1903 SYSCTL_PROC(_kern_tfp, KERN_TFP_POLICY, policy, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
1904 &tfp_policy, sizeof(uint32_t), &sysctl_settfp_policy, "I", "policy");
1905
1906 SYSCTL_INT(_vm, OID_AUTO, shared_region_trace_level, CTLFLAG_RW | CTLFLAG_LOCKED,
1907 &shared_region_trace_level, 0, "");
1908 SYSCTL_INT(_vm, OID_AUTO, shared_region_version, CTLFLAG_RD | CTLFLAG_LOCKED,
1909 &shared_region_version, 0, "");
1910 SYSCTL_INT(_vm, OID_AUTO, shared_region_persistence, CTLFLAG_RW | CTLFLAG_LOCKED,
1911 &shared_region_persistence, 0, "");
1912
1913 /*
1914 * shared_region_check_np:
1915 *
1916 * This system call is intended for dyld.
1917 *
1918 * dyld calls this when any process starts to see if the process's shared
1919 * region is already set up and ready to use.
1920 * This call returns the base address of the first mapping in the
1921 * process's shared region's first mapping.
1922 * dyld will then check what's mapped at that address.
1923 *
1924 * If the shared region is empty, dyld will then attempt to map the shared
1925 * cache file in the shared region via the shared_region_map_np() system call.
1926 *
1927 * If something's already mapped in the shared region, dyld will check if it
1928 * matches the shared cache it would like to use for that process.
1929 * If it matches, evrything's ready and the process can proceed and use the
1930 * shared region.
1931 * If it doesn't match, dyld will unmap the shared region and map the shared
1932 * cache into the process's address space via mmap().
1933 *
1934 * ERROR VALUES
1935 * EINVAL no shared region
1936 * ENOMEM shared region is empty
1937 * EFAULT bad address for "start_address"
1938 */
1939 int
1940 shared_region_check_np(
1941 __unused struct proc *p,
1942 struct shared_region_check_np_args *uap,
1943 __unused int *retvalp)
1944 {
1945 vm_shared_region_t shared_region;
1946 mach_vm_offset_t start_address = 0;
1947 int error = 0;
1948 kern_return_t kr;
1949
1950 SHARED_REGION_TRACE_DEBUG(
1951 ("shared_region: %p [%d(%s)] -> check_np(0x%llx)\n",
1952 (void *)VM_KERNEL_ADDRPERM(current_thread()),
1953 p->p_pid, p->p_comm,
1954 (uint64_t)uap->start_address));
1955
1956 /* retrieve the current tasks's shared region */
1957 shared_region = vm_shared_region_get(current_task());
1958 if (shared_region != NULL) {
1959 /* retrieve address of its first mapping... */
1960 kr = vm_shared_region_start_address(shared_region, &start_address);
1961 if (kr != KERN_SUCCESS) {
1962 error = ENOMEM;
1963 } else {
1964 #if __has_feature(ptrauth_calls)
1965 /*
1966 * Remap any section of the shared library that
1967 * has authenticated pointers into private memory.
1968 */
1969 if (vm_shared_region_auth_remap(shared_region) != KERN_SUCCESS) {
1970 error = ENOMEM;
1971 }
1972 #endif /* __has_feature(ptrauth_calls) */
1973
1974 /* ... and give it to the caller */
1975 if (error == 0) {
1976 error = copyout(&start_address,
1977 (user_addr_t) uap->start_address,
1978 sizeof(start_address));
1979 }
1980 if (error != 0) {
1981 SHARED_REGION_TRACE_ERROR(
1982 ("shared_region: %p [%d(%s)] "
1983 "check_np(0x%llx) "
1984 "copyout(0x%llx) error %d\n",
1985 (void *)VM_KERNEL_ADDRPERM(current_thread()),
1986 p->p_pid, p->p_comm,
1987 (uint64_t)uap->start_address, (uint64_t)start_address,
1988 error));
1989 }
1990 }
1991 vm_shared_region_deallocate(shared_region);
1992 } else {
1993 /* no shared region ! */
1994 error = EINVAL;
1995 }
1996
1997 SHARED_REGION_TRACE_DEBUG(
1998 ("shared_region: %p [%d(%s)] check_np(0x%llx) <- 0x%llx %d\n",
1999 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2000 p->p_pid, p->p_comm,
2001 (uint64_t)uap->start_address, (uint64_t)start_address, error));
2002
2003 return error;
2004 }
2005
2006
2007 static int
2008 shared_region_copyin(
2009 struct proc *p,
2010 user_addr_t user_addr,
2011 unsigned int count,
2012 unsigned int element_size,
2013 void *kernel_data)
2014 {
2015 int error = 0;
2016 vm_size_t size = count * element_size;
2017
2018 error = copyin(user_addr, kernel_data, size);
2019 if (error) {
2020 SHARED_REGION_TRACE_ERROR(
2021 ("shared_region: %p [%d(%s)] map(): "
2022 "copyin(0x%llx, %ld) failed (error=%d)\n",
2023 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2024 p->p_pid, p->p_comm,
2025 (uint64_t)user_addr, (long)size, error));
2026 }
2027 return error;
2028 }
2029
2030 #define _SR_FILE_MAPPINGS_MAX_FILES 2
2031
2032 /* forward declaration */
2033 __attribute__((noinline))
2034 static void shared_region_map_and_slide_cleanup(
2035 struct proc *p,
2036 uint32_t files_count,
2037 struct _sr_file_mappings *sr_file_mappings,
2038 struct vm_shared_region *shared_region,
2039 struct vnode *scdir_vp);
2040
2041 /*
2042 * Setup part of _shared_region_map_and_slide().
2043 * It had to be broken out of _shared_region_map_and_slide() to
2044 * prevent compiler inlining from blowing out the stack.
2045 */
2046 __attribute__((noinline))
2047 static int
2048 shared_region_map_and_slide_setup(
2049 struct proc *p,
2050 uint32_t files_count,
2051 struct shared_file_np *files,
2052 uint32_t mappings_count,
2053 struct shared_file_mapping_slide_np *mappings,
2054 struct _sr_file_mappings **sr_file_mappings,
2055 struct vm_shared_region **shared_region,
2056 struct vnode **scdir_vp)
2057 {
2058 int error = 0;
2059 struct _sr_file_mappings *srfmp;
2060 uint32_t mappings_next;
2061 struct vnode_attr va;
2062 off_t fs;
2063 #if CONFIG_MACF
2064 vm_prot_t maxprot = VM_PROT_ALL;
2065 #endif
2066 uint32_t i;
2067
2068 SHARED_REGION_TRACE_DEBUG(
2069 ("shared_region: %p [%d(%s)] -> map\n",
2070 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2071 p->p_pid, p->p_comm));
2072
2073 if (files_count > _SR_FILE_MAPPINGS_MAX_FILES) {
2074 error = E2BIG;
2075 goto done;
2076 }
2077 if (files_count == 0) {
2078 error = EINVAL;
2079 goto done;
2080 }
2081 *sr_file_mappings = kheap_alloc(KHEAP_TEMP, files_count * sizeof(struct _sr_file_mappings), Z_WAITOK);
2082 if (*sr_file_mappings == NULL) {
2083 error = ENOMEM;
2084 goto done;
2085 }
2086 bzero(*sr_file_mappings, files_count * sizeof(struct _sr_file_mappings));
2087 mappings_next = 0;
2088 for (i = 0; i < files_count; i++) {
2089 srfmp = &(*sr_file_mappings)[i];
2090 srfmp->fd = files[i].sf_fd;
2091 srfmp->mappings_count = files[i].sf_mappings_count;
2092 srfmp->mappings = &mappings[mappings_next];
2093 mappings_next += srfmp->mappings_count;
2094 if (mappings_next > mappings_count) {
2095 error = EINVAL;
2096 goto done;
2097 }
2098 srfmp->slide = files[i].sf_slide;
2099 }
2100
2101 if (scdir_enforce) {
2102 /* get vnode for scdir_path */
2103 error = vnode_lookup(scdir_path, 0, scdir_vp, vfs_context_current());
2104 if (error) {
2105 SHARED_REGION_TRACE_ERROR(
2106 ("shared_region: %p [%d(%s)]: "
2107 "vnode_lookup(%s) failed (error=%d)\n",
2108 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2109 p->p_pid, p->p_comm,
2110 scdir_path, error));
2111 goto done;
2112 }
2113 }
2114
2115 /* get the process's shared region (setup in vm_map_exec()) */
2116 *shared_region = vm_shared_region_trim_and_get(current_task());
2117 if (*shared_region == NULL) {
2118 SHARED_REGION_TRACE_ERROR(
2119 ("shared_region: %p [%d(%s)] map(): "
2120 "no shared region\n",
2121 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2122 p->p_pid, p->p_comm));
2123 error = EINVAL;
2124 goto done;
2125 }
2126
2127 for (srfmp = &(*sr_file_mappings)[0];
2128 srfmp < &(*sr_file_mappings)[files_count];
2129 srfmp++) {
2130 if (srfmp->mappings_count == 0) {
2131 /* no mappings here... */
2132 continue;
2133 }
2134
2135 /* get file structure from file descriptor */
2136 error = fp_get_ftype(p, srfmp->fd, DTYPE_VNODE, EINVAL, &srfmp->fp);
2137 if (error) {
2138 SHARED_REGION_TRACE_ERROR(
2139 ("shared_region: %p [%d(%s)] map: "
2140 "fd=%d lookup failed (error=%d)\n",
2141 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2142 p->p_pid, p->p_comm, srfmp->fd, error));
2143 goto done;
2144 }
2145
2146 /* we need at least read permission on the file */
2147 if (!(srfmp->fp->fp_glob->fg_flag & FREAD)) {
2148 SHARED_REGION_TRACE_ERROR(
2149 ("shared_region: %p [%d(%s)] map: "
2150 "fd=%d not readable\n",
2151 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2152 p->p_pid, p->p_comm, srfmp->fd));
2153 error = EPERM;
2154 goto done;
2155 }
2156
2157 /* get vnode from file structure */
2158 error = vnode_getwithref((vnode_t) srfmp->fp->fp_glob->fg_data);
2159 if (error) {
2160 SHARED_REGION_TRACE_ERROR(
2161 ("shared_region: %p [%d(%s)] map: "
2162 "fd=%d getwithref failed (error=%d)\n",
2163 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2164 p->p_pid, p->p_comm, srfmp->fd, error));
2165 goto done;
2166 }
2167 srfmp->vp = (struct vnode *) srfmp->fp->fp_glob->fg_data;
2168
2169 /* make sure the vnode is a regular file */
2170 if (srfmp->vp->v_type != VREG) {
2171 SHARED_REGION_TRACE_ERROR(
2172 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2173 "not a file (type=%d)\n",
2174 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2175 p->p_pid, p->p_comm,
2176 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2177 srfmp->vp->v_name, srfmp->vp->v_type));
2178 error = EINVAL;
2179 goto done;
2180 }
2181
2182 #if CONFIG_MACF
2183 /* pass in 0 for the offset argument because AMFI does not need the offset
2184 * of the shared cache */
2185 error = mac_file_check_mmap(vfs_context_ucred(vfs_context_current()),
2186 srfmp->fp->fp_glob, VM_PROT_ALL, MAP_FILE, 0, &maxprot);
2187 if (error) {
2188 goto done;
2189 }
2190 #endif /* MAC */
2191
2192 #if XNU_TARGET_OS_OSX && defined(__arm64__)
2193 /*
2194 * Check if the shared cache is in the trust cache;
2195 * if so, we can skip the root ownership check.
2196 */
2197 #if DEVELOPMENT || DEBUG
2198 /*
2199 * Skip both root ownership and trust cache check if
2200 * enforcement is disabled.
2201 */
2202 if (!cs_system_enforcement()) {
2203 goto after_root_check;
2204 }
2205 #endif /* DEVELOPMENT || DEBUG */
2206 struct cs_blob *blob = csvnode_get_blob(srfmp->vp, 0);
2207 if (blob == NULL) {
2208 SHARED_REGION_TRACE_ERROR(
2209 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2210 "missing CS blob\n",
2211 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2212 p->p_pid, p->p_comm,
2213 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2214 srfmp->vp->v_name));
2215 goto root_check;
2216 }
2217 const uint8_t *cdhash = csblob_get_cdhash(blob);
2218 if (cdhash == NULL) {
2219 SHARED_REGION_TRACE_ERROR(
2220 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2221 "missing cdhash\n",
2222 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2223 p->p_pid, p->p_comm,
2224 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2225 srfmp->vp->v_name));
2226 goto root_check;
2227 }
2228 uint32_t result = pmap_lookup_in_static_trust_cache(cdhash);
2229 boolean_t in_trust_cache = result & (TC_LOOKUP_FOUND << TC_LOOKUP_RESULT_SHIFT);
2230 if (!in_trust_cache) {
2231 SHARED_REGION_TRACE_ERROR(
2232 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2233 "not in trust cache\n",
2234 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2235 p->p_pid, p->p_comm,
2236 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2237 srfmp->vp->v_name));
2238 goto root_check;
2239 }
2240 goto after_root_check;
2241 root_check:
2242 #endif /* XNU_TARGET_OS_OSX && defined(__arm64__) */
2243
2244 /* The shared cache file must be owned by root */
2245 VATTR_INIT(&va);
2246 VATTR_WANTED(&va, va_uid);
2247 error = vnode_getattr(srfmp->vp, &va, vfs_context_current());
2248 if (error) {
2249 SHARED_REGION_TRACE_ERROR(
2250 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2251 "vnode_getattr(%p) failed (error=%d)\n",
2252 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2253 p->p_pid, p->p_comm,
2254 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2255 srfmp->vp->v_name,
2256 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2257 error));
2258 goto done;
2259 }
2260 if (va.va_uid != 0) {
2261 SHARED_REGION_TRACE_ERROR(
2262 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2263 "owned by uid=%d instead of 0\n",
2264 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2265 p->p_pid, p->p_comm,
2266 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2267 srfmp->vp->v_name, va.va_uid));
2268 error = EPERM;
2269 goto done;
2270 }
2271
2272 #if XNU_TARGET_OS_OSX && defined(__arm64__)
2273 after_root_check:
2274 #endif /* XNU_TARGET_OS_OSX && defined(__arm64__) */
2275
2276 #if CONFIG_CSR
2277 if (csr_check(CSR_ALLOW_UNRESTRICTED_FS) != 0) {
2278 VATTR_INIT(&va);
2279 VATTR_WANTED(&va, va_flags);
2280 error = vnode_getattr(srfmp->vp, &va, vfs_context_current());
2281 if (error) {
2282 SHARED_REGION_TRACE_ERROR(
2283 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2284 "vnode_getattr(%p) failed (error=%d)\n",
2285 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2286 p->p_pid, p->p_comm,
2287 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2288 srfmp->vp->v_name,
2289 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2290 error));
2291 goto done;
2292 }
2293
2294 if (!(va.va_flags & SF_RESTRICTED)) {
2295 /*
2296 * CSR is not configured in CSR_ALLOW_UNRESTRICTED_FS mode, and
2297 * the shared cache file is NOT SIP-protected, so reject the
2298 * mapping request
2299 */
2300 SHARED_REGION_TRACE_ERROR(
2301 ("shared_region: %p [%d(%s)] map(%p:'%s'), "
2302 "vnode is not SIP-protected. \n",
2303 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2304 p->p_pid, p->p_comm,
2305 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2306 srfmp->vp->v_name));
2307 error = EPERM;
2308 goto done;
2309 }
2310 }
2311 #else /* CONFIG_CSR */
2312 /* Devices without SIP/ROSP need to make sure that the shared cache is on the root volume. */
2313
2314 struct vnode *root_vp = p->p_fd->fd_rdir;
2315 if (root_vp == NULL) {
2316 root_vp = rootvnode;
2317 }
2318 if (srfmp->vp->v_mount != root_vp->v_mount) {
2319 SHARED_REGION_TRACE_ERROR(
2320 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2321 "not on process's root volume\n",
2322 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2323 p->p_pid, p->p_comm,
2324 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2325 srfmp->vp->v_name));
2326 error = EPERM;
2327 goto done;
2328 }
2329 #endif /* CONFIG_CSR */
2330
2331 if (scdir_enforce) {
2332 /* ensure parent is scdir_vp */
2333 assert(*scdir_vp != NULL);
2334 if (vnode_parent(srfmp->vp) != *scdir_vp) {
2335 SHARED_REGION_TRACE_ERROR(
2336 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2337 "shared cache file not in %s\n",
2338 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2339 p->p_pid, p->p_comm,
2340 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2341 srfmp->vp->v_name, scdir_path));
2342 error = EPERM;
2343 goto done;
2344 }
2345 }
2346
2347 /* get vnode size */
2348 error = vnode_size(srfmp->vp, &fs, vfs_context_current());
2349 if (error) {
2350 SHARED_REGION_TRACE_ERROR(
2351 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2352 "vnode_size(%p) failed (error=%d)\n",
2353 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2354 p->p_pid, p->p_comm,
2355 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2356 srfmp->vp->v_name,
2357 (void *)VM_KERNEL_ADDRPERM(srfmp->vp), error));
2358 goto done;
2359 }
2360 srfmp->file_size = fs;
2361
2362 /* get the file's memory object handle */
2363 srfmp->file_control = ubc_getobject(srfmp->vp, UBC_HOLDOBJECT);
2364 if (srfmp->file_control == MEMORY_OBJECT_CONTROL_NULL) {
2365 SHARED_REGION_TRACE_ERROR(
2366 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2367 "no memory object\n",
2368 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2369 p->p_pid, p->p_comm,
2370 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2371 srfmp->vp->v_name));
2372 error = EINVAL;
2373 goto done;
2374 }
2375
2376 /* check that the mappings are properly covered by code signatures */
2377 if (!cs_system_enforcement()) {
2378 /* code signing is not enforced: no need to check */
2379 } else {
2380 for (i = 0; i < srfmp->mappings_count; i++) {
2381 if (srfmp->mappings[i].sms_init_prot & VM_PROT_ZF) {
2382 /* zero-filled mapping: not backed by the file */
2383 continue;
2384 }
2385 if (ubc_cs_is_range_codesigned(srfmp->vp,
2386 srfmp->mappings[i].sms_file_offset,
2387 srfmp->mappings[i].sms_size)) {
2388 /* this mapping is fully covered by code signatures */
2389 continue;
2390 }
2391 SHARED_REGION_TRACE_ERROR(
2392 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2393 "mapping #%d/%d [0x%llx:0x%llx:0x%llx:0x%x:0x%x] "
2394 "is not code-signed\n",
2395 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2396 p->p_pid, p->p_comm,
2397 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2398 srfmp->vp->v_name,
2399 i, srfmp->mappings_count,
2400 srfmp->mappings[i].sms_address,
2401 srfmp->mappings[i].sms_size,
2402 srfmp->mappings[i].sms_file_offset,
2403 srfmp->mappings[i].sms_max_prot,
2404 srfmp->mappings[i].sms_init_prot));
2405 error = EINVAL;
2406 goto done;
2407 }
2408 }
2409 }
2410 done:
2411 if (error != 0) {
2412 shared_region_map_and_slide_cleanup(p, files_count, *sr_file_mappings, *shared_region, *scdir_vp);
2413 *sr_file_mappings = NULL;
2414 *shared_region = NULL;
2415 *scdir_vp = NULL;
2416 }
2417 return error;
2418 }
2419
2420 /*
2421 * shared_region_map_np()
2422 *
2423 * This system call is intended for dyld.
2424 *
2425 * dyld uses this to map a shared cache file into a shared region.
2426 * This is usually done only the first time a shared cache is needed.
2427 * Subsequent processes will just use the populated shared region without
2428 * requiring any further setup.
2429 */
2430 static int
2431 _shared_region_map_and_slide(
2432 struct proc *p,
2433 uint32_t files_count,
2434 struct shared_file_np *files,
2435 uint32_t mappings_count,
2436 struct shared_file_mapping_slide_np *mappings)
2437 {
2438 int error = 0;
2439 kern_return_t kr = KERN_SUCCESS;
2440 struct _sr_file_mappings *sr_file_mappings = NULL;
2441 struct vnode *scdir_vp = NULL;
2442 struct vm_shared_region *shared_region = NULL;
2443
2444 /*
2445 * Turn files, mappings into sr_file_mappings and other setup.
2446 */
2447 error = shared_region_map_and_slide_setup(p, files_count,
2448 files, mappings_count, mappings,
2449 &sr_file_mappings, &shared_region, &scdir_vp);
2450 if (error != 0) {
2451 return error;
2452 }
2453
2454 /* map the file(s) into that shared region's submap */
2455 kr = vm_shared_region_map_file(shared_region,
2456 (void *) p->p_fd->fd_rdir,
2457 files_count,
2458 sr_file_mappings);
2459 if (kr != KERN_SUCCESS) {
2460 SHARED_REGION_TRACE_ERROR(("shared_region: %p [%d(%s)] map(): "
2461 "vm_shared_region_map_file() failed kr=0x%x\n",
2462 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2463 p->p_pid, p->p_comm, kr));
2464 }
2465
2466 /* convert kern_return_t to errno */
2467 switch (kr) {
2468 case KERN_SUCCESS:
2469 error = 0;
2470 break;
2471 case KERN_INVALID_ADDRESS:
2472 error = EFAULT;
2473 break;
2474 case KERN_PROTECTION_FAILURE:
2475 error = EPERM;
2476 break;
2477 case KERN_NO_SPACE:
2478 error = ENOMEM;
2479 break;
2480 case KERN_FAILURE:
2481 case KERN_INVALID_ARGUMENT:
2482 default:
2483 error = EINVAL;
2484 break;
2485 }
2486
2487 /*
2488 * Mark that this process is now using split libraries.
2489 */
2490 if (error == 0 && (p->p_flag & P_NOSHLIB)) {
2491 OSBitAndAtomic(~((uint32_t)P_NOSHLIB), &p->p_flag);
2492 }
2493
2494 shared_region_map_and_slide_cleanup(p, files_count, sr_file_mappings, shared_region, scdir_vp);
2495
2496 SHARED_REGION_TRACE_DEBUG(
2497 ("shared_region: %p [%d(%s)] <- map\n",
2498 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2499 p->p_pid, p->p_comm));
2500
2501 return error;
2502 }
2503
2504 /*
2505 * Clean up part of _shared_region_map_and_slide()
2506 * It had to be broken out of _shared_region_map_and_slide() to
2507 * prevent compiler inlining from blowing out the stack.
2508 */
2509 __attribute__((noinline))
2510 static void
2511 shared_region_map_and_slide_cleanup(
2512 struct proc *p,
2513 uint32_t files_count,
2514 struct _sr_file_mappings *sr_file_mappings,
2515 struct vm_shared_region *shared_region,
2516 struct vnode *scdir_vp)
2517 {
2518 struct _sr_file_mappings *srfmp;
2519 struct vnode_attr va;
2520
2521 if (sr_file_mappings != NULL) {
2522 for (srfmp = &sr_file_mappings[0]; srfmp < &sr_file_mappings[files_count]; srfmp++) {
2523 if (srfmp->vp != NULL) {
2524 vnode_lock_spin(srfmp->vp);
2525 srfmp->vp->v_flag |= VSHARED_DYLD;
2526 vnode_unlock(srfmp->vp);
2527
2528 /* update the vnode's access time */
2529 if (!(vnode_vfsvisflags(srfmp->vp) & MNT_NOATIME)) {
2530 VATTR_INIT(&va);
2531 nanotime(&va.va_access_time);
2532 VATTR_SET_ACTIVE(&va, va_access_time);
2533 vnode_setattr(srfmp->vp, &va, vfs_context_current());
2534 }
2535
2536 #if NAMEDSTREAMS
2537 /*
2538 * If the shared cache is compressed, it may
2539 * have a namedstream vnode instantiated for
2540 * for it. That namedstream vnode will also
2541 * have to be marked with VSHARED_DYLD.
2542 */
2543 if (vnode_hasnamedstreams(srfmp->vp)) {
2544 vnode_t svp;
2545 if (vnode_getnamedstream(srfmp->vp, &svp, XATTR_RESOURCEFORK_NAME,
2546 NS_OPEN, 0, vfs_context_kernel()) == 0) {
2547 vnode_lock_spin(svp);
2548 svp->v_flag |= VSHARED_DYLD;
2549 vnode_unlock(svp);
2550 vnode_put(svp);
2551 }
2552 }
2553 #endif /* NAMEDSTREAMS */
2554 /*
2555 * release the vnode...
2556 * ubc_map() still holds it for us in the non-error case
2557 */
2558 (void) vnode_put(srfmp->vp);
2559 srfmp->vp = NULL;
2560 }
2561 if (srfmp->fp != NULL) {
2562 /* release the file descriptor */
2563 fp_drop(p, srfmp->fd, srfmp->fp, 0);
2564 srfmp->fp = NULL;
2565 }
2566 }
2567 kheap_free(KHEAP_TEMP, sr_file_mappings, files_count * sizeof(*sr_file_mappings));
2568 }
2569
2570 if (scdir_vp != NULL) {
2571 (void)vnode_put(scdir_vp);
2572 scdir_vp = NULL;
2573 }
2574
2575 if (shared_region != NULL) {
2576 vm_shared_region_deallocate(shared_region);
2577 }
2578 }
2579
2580
2581 #define SFM_MAX 1024 /* max mapping structs allowed to pass in */
2582
2583 /*
2584 * This interface is used by dyld to map shared caches which are
2585 * for any architecture which doesn't have run time support of pointer
2586 * authentication. Note dyld could also use the new ...map_and_slide_2_np()
2587 * call for this case, however, it just doesn't do that yet.
2588 */
2589 int
2590 shared_region_map_and_slide_np(
2591 struct proc *p,
2592 struct shared_region_map_and_slide_np_args *uap,
2593 __unused int *retvalp)
2594 {
2595 unsigned int mappings_count = uap->count;
2596 unsigned int m;
2597 uint32_t slide = uap->slide;
2598 struct shared_file_np shared_files[1];
2599 struct shared_file_mapping_np legacy_mapping;
2600 struct shared_file_mapping_slide_np *mappings = NULL;
2601 kern_return_t kr = KERN_SUCCESS;
2602
2603 if ((kr = vm_shared_region_sliding_valid(slide)) != KERN_SUCCESS) {
2604 if (kr == KERN_INVALID_ARGUMENT) {
2605 /*
2606 * This will happen if we request sliding again
2607 * with the same slide value that was used earlier
2608 * for the very first sliding.
2609 */
2610 kr = KERN_SUCCESS;
2611 }
2612 goto done;
2613 }
2614
2615 if (mappings_count == 0) {
2616 SHARED_REGION_TRACE_INFO(
2617 ("shared_region: %p [%d(%s)] map(): "
2618 "no mappings\n",
2619 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2620 p->p_pid, p->p_comm));
2621 kr = 0; /* no mappings: we're done ! */
2622 goto done;
2623 } else if (mappings_count <= SFM_MAX) {
2624 mappings = kheap_alloc(KHEAP_TEMP,
2625 mappings_count * sizeof(mappings[0]), Z_WAITOK);
2626 if (mappings == NULL) {
2627 kr = KERN_RESOURCE_SHORTAGE;
2628 goto done;
2629 }
2630 } else {
2631 SHARED_REGION_TRACE_ERROR(
2632 ("shared_region: %p [%d(%s)] map(): "
2633 "too many mappings (%d) max %d\n",
2634 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2635 p->p_pid, p->p_comm,
2636 mappings_count, SFM_MAX));
2637 kr = KERN_FAILURE;
2638 goto done;
2639 }
2640
2641 /*
2642 * Read in the mappings and translate to new format.
2643 */
2644 for (m = 0; m < mappings_count; ++m) {
2645 user_addr_t from_uaddr = uap->mappings + (m * sizeof(struct shared_file_mapping_np));
2646 kr = shared_region_copyin(p, from_uaddr, 1, sizeof(legacy_mapping), &legacy_mapping);
2647 if (kr != 0) {
2648 goto done;
2649 }
2650 mappings[m].sms_address = legacy_mapping.sfm_address;
2651 mappings[m].sms_size = legacy_mapping.sfm_size;
2652 mappings[m].sms_file_offset = legacy_mapping.sfm_file_offset;
2653 mappings[m].sms_max_prot = legacy_mapping.sfm_max_prot;
2654 mappings[m].sms_init_prot = legacy_mapping.sfm_init_prot;
2655 mappings[m].sms_slide_size = uap->slide_size;
2656 mappings[m].sms_slide_start = uap->slide_start;
2657 }
2658
2659 bzero(shared_files, sizeof(shared_files));
2660 shared_files[0].sf_fd = uap->fd;
2661 shared_files[0].sf_mappings_count = mappings_count;
2662 shared_files[0].sf_slide = slide;
2663
2664 kr = _shared_region_map_and_slide(p,
2665 1, /* # of files to map */
2666 &shared_files[0], /* files to map */
2667 mappings_count,
2668 mappings);
2669
2670 done:
2671 if (mappings != NULL) {
2672 kheap_free(KHEAP_TEMP, mappings, mappings_count * sizeof(mappings[0]));
2673 mappings = NULL;
2674 }
2675 return kr;
2676 }
2677
2678 /*
2679 * This interface for setting up shared region mappings is what dyld
2680 * uses for shared caches that have __AUTH sections. All other shared
2681 * caches use the non _2 version.
2682 *
2683 * The slide used for shared regions setup using this interface is done differently
2684 * from the old interface. The slide value passed in the shared_files_np represents
2685 * a max value. The kernel will choose a random value based on that, then use it
2686 * for all shared regions.
2687 */
2688 #define SLIDE_AMOUNT_MASK ~PAGE_MASK
2689
2690 int
2691 shared_region_map_and_slide_2_np(
2692 struct proc *p,
2693 struct shared_region_map_and_slide_2_np_args *uap,
2694 __unused int *retvalp)
2695 {
2696 unsigned int files_count;
2697 struct shared_file_np *shared_files = NULL;
2698 unsigned int mappings_count;
2699 struct shared_file_mapping_slide_np *mappings = NULL;
2700 kern_return_t kr = KERN_SUCCESS;
2701 boolean_t should_slide_mappings = TRUE;
2702
2703 files_count = uap->files_count;
2704 mappings_count = uap->mappings_count;
2705
2706
2707 if (files_count == 0) {
2708 SHARED_REGION_TRACE_INFO(
2709 ("shared_region: %p [%d(%s)] map(): "
2710 "no files\n",
2711 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2712 p->p_pid, p->p_comm));
2713 kr = 0; /* no files to map: we're done ! */
2714 goto done;
2715 } else if (files_count <= _SR_FILE_MAPPINGS_MAX_FILES) {
2716 shared_files = kheap_alloc(KHEAP_TEMP,
2717 files_count * sizeof(shared_files[0]), Z_WAITOK);
2718 if (shared_files == NULL) {
2719 kr = KERN_RESOURCE_SHORTAGE;
2720 goto done;
2721 }
2722 } else {
2723 SHARED_REGION_TRACE_ERROR(
2724 ("shared_region: %p [%d(%s)] map(): "
2725 "too many files (%d) max %d\n",
2726 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2727 p->p_pid, p->p_comm,
2728 files_count, _SR_FILE_MAPPINGS_MAX_FILES));
2729 kr = KERN_FAILURE;
2730 goto done;
2731 }
2732
2733 if (mappings_count == 0) {
2734 SHARED_REGION_TRACE_INFO(
2735 ("shared_region: %p [%d(%s)] map(): "
2736 "no mappings\n",
2737 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2738 p->p_pid, p->p_comm));
2739 kr = 0; /* no mappings: we're done ! */
2740 goto done;
2741 } else if (mappings_count <= SFM_MAX) {
2742 mappings = kheap_alloc(KHEAP_TEMP,
2743 mappings_count * sizeof(mappings[0]), Z_WAITOK);
2744 if (mappings == NULL) {
2745 kr = KERN_RESOURCE_SHORTAGE;
2746 goto done;
2747 }
2748 } else {
2749 SHARED_REGION_TRACE_ERROR(
2750 ("shared_region: %p [%d(%s)] map(): "
2751 "too many mappings (%d) max %d\n",
2752 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2753 p->p_pid, p->p_comm,
2754 mappings_count, SFM_MAX));
2755 kr = KERN_FAILURE;
2756 goto done;
2757 }
2758
2759 kr = shared_region_copyin(p, uap->files, files_count, sizeof(shared_files[0]), shared_files);
2760 if (kr != KERN_SUCCESS) {
2761 goto done;
2762 }
2763
2764 kr = shared_region_copyin(p, uap->mappings, mappings_count, sizeof(mappings[0]), mappings);
2765 if (kr != KERN_SUCCESS) {
2766 goto done;
2767 }
2768
2769 if (should_slide_mappings) {
2770 uint32_t max_slide = shared_files[0].sf_slide;
2771 uint32_t random_val;
2772 uint32_t slide_amount;
2773
2774 if (max_slide != 0) {
2775 read_random(&random_val, sizeof random_val);
2776 slide_amount = ((random_val % max_slide) & SLIDE_AMOUNT_MASK);
2777 } else {
2778 slide_amount = 0;
2779 }
2780
2781 /*
2782 * Fix up the mappings to reflect the desired slide.
2783 */
2784 unsigned int f;
2785 unsigned int m = 0;
2786 unsigned int i;
2787 for (f = 0; f < files_count; ++f) {
2788 shared_files[f].sf_slide = slide_amount;
2789 for (i = 0; i < shared_files[f].sf_mappings_count; ++i, ++m) {
2790 if (m >= mappings_count) {
2791 SHARED_REGION_TRACE_ERROR(
2792 ("shared_region: %p [%d(%s)] map(): "
2793 "mapping count argument was too small\n",
2794 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2795 p->p_pid, p->p_comm));
2796 kr = KERN_FAILURE;
2797 goto done;
2798 }
2799 mappings[m].sms_address += slide_amount;
2800 if (mappings[m].sms_slide_size != 0) {
2801 mappings[i].sms_slide_start += slide_amount;
2802 }
2803 }
2804 }
2805 }
2806 kr = _shared_region_map_and_slide(p, files_count, shared_files, mappings_count, mappings);
2807 done:
2808 if (shared_files != NULL) {
2809 kheap_free(KHEAP_TEMP, shared_files, files_count * sizeof(shared_files[0]));
2810 shared_files = NULL;
2811 }
2812 if (mappings != NULL) {
2813 kheap_free(KHEAP_TEMP, mappings, mappings_count * sizeof(mappings[0]));
2814 mappings = NULL;
2815 }
2816 return kr;
2817 }
2818
2819 /* sysctl overflow room */
2820
2821 SYSCTL_INT(_vm, OID_AUTO, pagesize, CTLFLAG_RD | CTLFLAG_LOCKED,
2822 (int *) &page_size, 0, "vm page size");
2823
2824 /* vm_page_free_target is provided as a makeshift solution for applications that want to
2825 * allocate buffer space, possibly purgeable memory, but not cause inactive pages to be
2826 * reclaimed. It allows the app to calculate how much memory is free outside the free target. */
2827 extern unsigned int vm_page_free_target;
2828 SYSCTL_INT(_vm, OID_AUTO, vm_page_free_target, CTLFLAG_RD | CTLFLAG_LOCKED,
2829 &vm_page_free_target, 0, "Pageout daemon free target");
2830
2831 SYSCTL_INT(_vm, OID_AUTO, memory_pressure, CTLFLAG_RD | CTLFLAG_LOCKED,
2832 &vm_pageout_state.vm_memory_pressure, 0, "Memory pressure indicator");
2833
2834 static int
2835 vm_ctl_page_free_wanted SYSCTL_HANDLER_ARGS
2836 {
2837 #pragma unused(oidp, arg1, arg2)
2838 unsigned int page_free_wanted;
2839
2840 page_free_wanted = mach_vm_ctl_page_free_wanted();
2841 return SYSCTL_OUT(req, &page_free_wanted, sizeof(page_free_wanted));
2842 }
2843 SYSCTL_PROC(_vm, OID_AUTO, page_free_wanted,
2844 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
2845 0, 0, vm_ctl_page_free_wanted, "I", "");
2846
2847 extern unsigned int vm_page_purgeable_count;
2848 SYSCTL_INT(_vm, OID_AUTO, page_purgeable_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2849 &vm_page_purgeable_count, 0, "Purgeable page count");
2850
2851 extern unsigned int vm_page_purgeable_wired_count;
2852 SYSCTL_INT(_vm, OID_AUTO, page_purgeable_wired_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2853 &vm_page_purgeable_wired_count, 0, "Wired purgeable page count");
2854
2855 extern unsigned int vm_page_kern_lpage_count;
2856 SYSCTL_INT(_vm, OID_AUTO, kern_lpage_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2857 &vm_page_kern_lpage_count, 0, "kernel used large pages");
2858
2859 #if DEVELOPMENT || DEBUG
2860 #if __ARM_MIXED_PAGE_SIZE__
2861 static int vm_mixed_pagesize_supported = 1;
2862 #else
2863 static int vm_mixed_pagesize_supported = 0;
2864 #endif /*__ARM_MIXED_PAGE_SIZE__ */
2865 SYSCTL_INT(_debug, OID_AUTO, vm_mixed_pagesize_supported, CTLFLAG_ANYBODY | CTLFLAG_RD | CTLFLAG_LOCKED,
2866 &vm_mixed_pagesize_supported, 0, "kernel support for mixed pagesize");
2867
2868
2869 extern uint64_t get_pages_grabbed_count(void);
2870
2871 static int
2872 pages_grabbed SYSCTL_HANDLER_ARGS
2873 {
2874 #pragma unused(arg1, arg2, oidp)
2875 uint64_t value = get_pages_grabbed_count();
2876 return SYSCTL_OUT(req, &value, sizeof(value));
2877 }
2878
2879 SYSCTL_PROC(_vm, OID_AUTO, pages_grabbed, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED,
2880 0, 0, &pages_grabbed, "QU", "Total pages grabbed");
2881 SYSCTL_ULONG(_vm, OID_AUTO, pages_freed, CTLFLAG_RD | CTLFLAG_LOCKED,
2882 &vm_pageout_vminfo.vm_page_pages_freed, "Total pages freed");
2883
2884 SYSCTL_INT(_vm, OID_AUTO, pageout_purged_objects, CTLFLAG_RD | CTLFLAG_LOCKED,
2885 &vm_pageout_debug.vm_pageout_purged_objects, 0, "System purged object count");
2886 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_busy, CTLFLAG_RD | CTLFLAG_LOCKED,
2887 &vm_pageout_debug.vm_pageout_cleaned_busy, 0, "Cleaned pages busy (deactivated)");
2888 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_nolock, CTLFLAG_RD | CTLFLAG_LOCKED,
2889 &vm_pageout_debug.vm_pageout_cleaned_nolock, 0, "Cleaned pages no-lock (deactivated)");
2890
2891 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_volatile_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2892 &vm_pageout_debug.vm_pageout_cleaned_volatile_reactivated, 0, "Cleaned pages volatile reactivated");
2893 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_fault_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2894 &vm_pageout_debug.vm_pageout_cleaned_fault_reactivated, 0, "Cleaned pages fault reactivated");
2895 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2896 &vm_pageout_debug.vm_pageout_cleaned_reactivated, 0, "Cleaned pages reactivated"); /* sum of all reactivated AND busy and nolock (even though those actually get reDEactivated */
2897 SYSCTL_ULONG(_vm, OID_AUTO, pageout_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED,
2898 &vm_pageout_vminfo.vm_pageout_freed_cleaned, "Cleaned pages freed");
2899 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_reference_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2900 &vm_pageout_debug.vm_pageout_cleaned_reference_reactivated, 0, "Cleaned pages reference reactivated");
2901 SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED,
2902 &vm_pageout_debug.vm_pageout_enqueued_cleaned, 0, ""); /* sum of next two */
2903 #endif /* DEVELOPMENT || DEBUG */
2904
2905 extern int madvise_free_debug;
2906 SYSCTL_INT(_vm, OID_AUTO, madvise_free_debug, CTLFLAG_RW | CTLFLAG_LOCKED,
2907 &madvise_free_debug, 0, "zero-fill on madvise(MADV_FREE*)");
2908
2909 SYSCTL_INT(_vm, OID_AUTO, page_reusable_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2910 &vm_page_stats_reusable.reusable_count, 0, "Reusable page count");
2911 SYSCTL_QUAD(_vm, OID_AUTO, reusable_success, CTLFLAG_RD | CTLFLAG_LOCKED,
2912 &vm_page_stats_reusable.reusable_pages_success, "");
2913 SYSCTL_QUAD(_vm, OID_AUTO, reusable_failure, CTLFLAG_RD | CTLFLAG_LOCKED,
2914 &vm_page_stats_reusable.reusable_pages_failure, "");
2915 SYSCTL_QUAD(_vm, OID_AUTO, reusable_pages_shared, CTLFLAG_RD | CTLFLAG_LOCKED,
2916 &vm_page_stats_reusable.reusable_pages_shared, "");
2917 SYSCTL_QUAD(_vm, OID_AUTO, all_reusable_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2918 &vm_page_stats_reusable.all_reusable_calls, "");
2919 SYSCTL_QUAD(_vm, OID_AUTO, partial_reusable_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2920 &vm_page_stats_reusable.partial_reusable_calls, "");
2921 SYSCTL_QUAD(_vm, OID_AUTO, reuse_success, CTLFLAG_RD | CTLFLAG_LOCKED,
2922 &vm_page_stats_reusable.reuse_pages_success, "");
2923 SYSCTL_QUAD(_vm, OID_AUTO, reuse_failure, CTLFLAG_RD | CTLFLAG_LOCKED,
2924 &vm_page_stats_reusable.reuse_pages_failure, "");
2925 SYSCTL_QUAD(_vm, OID_AUTO, all_reuse_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2926 &vm_page_stats_reusable.all_reuse_calls, "");
2927 SYSCTL_QUAD(_vm, OID_AUTO, partial_reuse_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2928 &vm_page_stats_reusable.partial_reuse_calls, "");
2929 SYSCTL_QUAD(_vm, OID_AUTO, can_reuse_success, CTLFLAG_RD | CTLFLAG_LOCKED,
2930 &vm_page_stats_reusable.can_reuse_success, "");
2931 SYSCTL_QUAD(_vm, OID_AUTO, can_reuse_failure, CTLFLAG_RD | CTLFLAG_LOCKED,
2932 &vm_page_stats_reusable.can_reuse_failure, "");
2933 SYSCTL_QUAD(_vm, OID_AUTO, reusable_reclaimed, CTLFLAG_RD | CTLFLAG_LOCKED,
2934 &vm_page_stats_reusable.reusable_reclaimed, "");
2935 SYSCTL_QUAD(_vm, OID_AUTO, reusable_nonwritable, CTLFLAG_RD | CTLFLAG_LOCKED,
2936 &vm_page_stats_reusable.reusable_nonwritable, "");
2937 SYSCTL_QUAD(_vm, OID_AUTO, reusable_shared, CTLFLAG_RD | CTLFLAG_LOCKED,
2938 &vm_page_stats_reusable.reusable_shared, "");
2939 SYSCTL_QUAD(_vm, OID_AUTO, free_shared, CTLFLAG_RD | CTLFLAG_LOCKED,
2940 &vm_page_stats_reusable.free_shared, "");
2941
2942
2943 extern unsigned int vm_page_free_count, vm_page_speculative_count;
2944 SYSCTL_UINT(_vm, OID_AUTO, page_free_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_free_count, 0, "");
2945 SYSCTL_UINT(_vm, OID_AUTO, page_speculative_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_speculative_count, 0, "");
2946
2947 extern unsigned int vm_page_cleaned_count;
2948 SYSCTL_UINT(_vm, OID_AUTO, page_cleaned_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_cleaned_count, 0, "Cleaned queue size");
2949
2950 extern unsigned int vm_page_pageable_internal_count, vm_page_pageable_external_count;
2951 SYSCTL_UINT(_vm, OID_AUTO, page_pageable_internal_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pageable_internal_count, 0, "");
2952 SYSCTL_UINT(_vm, OID_AUTO, page_pageable_external_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pageable_external_count, 0, "");
2953
2954 /* pageout counts */
2955 SYSCTL_UINT(_vm, OID_AUTO, pageout_inactive_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_state.vm_pageout_inactive_clean, 0, "");
2956 SYSCTL_UINT(_vm, OID_AUTO, pageout_inactive_used, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_state.vm_pageout_inactive_used, 0, "");
2957
2958 SYSCTL_ULONG(_vm, OID_AUTO, pageout_inactive_dirty_internal, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_inactive_dirty_internal, "");
2959 SYSCTL_ULONG(_vm, OID_AUTO, pageout_inactive_dirty_external, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_inactive_dirty_external, "");
2960 SYSCTL_ULONG(_vm, OID_AUTO, pageout_speculative_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_speculative, "");
2961 SYSCTL_ULONG(_vm, OID_AUTO, pageout_freed_external, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_external, "");
2962 SYSCTL_ULONG(_vm, OID_AUTO, pageout_freed_speculative, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_speculative, "");
2963 SYSCTL_ULONG(_vm, OID_AUTO, pageout_freed_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_cleaned, "");
2964
2965
2966 /* counts of pages prefaulted when entering a memory object */
2967 extern int64_t vm_prefault_nb_pages, vm_prefault_nb_bailout;
2968 SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_pages, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_pages, "");
2969 SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_bailout, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_bailout, "");
2970
2971 #if defined (__x86_64__)
2972 extern unsigned int vm_clump_promote_threshold;
2973 SYSCTL_UINT(_vm, OID_AUTO, vm_clump_promote_threshold, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_clump_promote_threshold, 0, "clump size threshold for promotes");
2974 #if DEVELOPMENT || DEBUG
2975 extern unsigned long vm_clump_stats[];
2976 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats1, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[1], "free page allocations from clump of 1 page");
2977 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats2, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[2], "free page allocations from clump of 2 pages");
2978 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats3, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[3], "free page allocations from clump of 3 pages");
2979 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats4, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[4], "free page allocations from clump of 4 pages");
2980 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats5, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[5], "free page allocations from clump of 5 pages");
2981 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats6, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[6], "free page allocations from clump of 6 pages");
2982 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats7, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[7], "free page allocations from clump of 7 pages");
2983 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats8, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[8], "free page allocations from clump of 8 pages");
2984 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats9, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[9], "free page allocations from clump of 9 pages");
2985 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats10, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[10], "free page allocations from clump of 10 pages");
2986 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats11, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[11], "free page allocations from clump of 11 pages");
2987 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats12, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[12], "free page allocations from clump of 12 pages");
2988 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats13, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[13], "free page allocations from clump of 13 pages");
2989 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats14, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[14], "free page allocations from clump of 14 pages");
2990 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats15, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[15], "free page allocations from clump of 15 pages");
2991 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats16, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[16], "free page allocations from clump of 16 pages");
2992 extern unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes;
2993 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_alloc, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_allocs, "free page allocations");
2994 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_inserts, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_inserts, "free page insertions");
2995 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_inrange, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_inrange, "free page insertions that are part of vm_pages");
2996 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_promotes, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_promotes, "pages promoted to head");
2997 #endif /* if DEVELOPMENT || DEBUG */
2998 #endif /* #if defined (__x86_64__) */
2999
3000 #if CONFIG_SECLUDED_MEMORY
3001
3002 SYSCTL_UINT(_vm, OID_AUTO, num_tasks_can_use_secluded_mem, CTLFLAG_RD | CTLFLAG_LOCKED, &num_tasks_can_use_secluded_mem, 0, "");
3003 extern unsigned int vm_page_secluded_target;
3004 extern unsigned int vm_page_secluded_count;
3005 extern unsigned int vm_page_secluded_count_free;
3006 extern unsigned int vm_page_secluded_count_inuse;
3007 extern unsigned int vm_page_secluded_count_over_target;
3008 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_target, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_target, 0, "");
3009 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count, 0, "");
3010 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count_free, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count_free, 0, "");
3011 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count_inuse, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count_inuse, 0, "");
3012 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count_over_target, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count_over_target, 0, "");
3013
3014 extern struct vm_page_secluded_data vm_page_secluded;
3015 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_eligible, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.eligible_for_secluded, 0, "");
3016 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_success_free, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_success_free, 0, "");
3017 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_success_other, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_success_other, 0, "");
3018 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_locked, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_failure_locked, 0, "");
3019 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_state, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_failure_state, 0, "");
3020 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_dirty, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_failure_dirty, 0, "");
3021 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_for_iokit, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_for_iokit, 0, "");
3022 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_for_iokit_success, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_for_iokit_success, 0, "");
3023
3024 #endif /* CONFIG_SECLUDED_MEMORY */
3025
3026 #include <kern/thread.h>
3027 #include <sys/user.h>
3028
3029 void vm_pageout_io_throttle(void);
3030
3031 void
3032 vm_pageout_io_throttle(void)
3033 {
3034 struct uthread *uthread = get_bsdthread_info(current_thread());
3035
3036 /*
3037 * thread is marked as a low priority I/O type
3038 * and the I/O we issued while in this cleaning operation
3039 * collided with normal I/O operations... we'll
3040 * delay in order to mitigate the impact of this
3041 * task on the normal operation of the system
3042 */
3043
3044 if (uthread->uu_lowpri_window) {
3045 throttle_lowpri_io(1);
3046 }
3047 }
3048
3049 int
3050 vm_pressure_monitor(
3051 __unused struct proc *p,
3052 struct vm_pressure_monitor_args *uap,
3053 int *retval)
3054 {
3055 kern_return_t kr;
3056 uint32_t pages_reclaimed;
3057 uint32_t pages_wanted;
3058
3059 kr = mach_vm_pressure_monitor(
3060 (boolean_t) uap->wait_for_pressure,
3061 uap->nsecs_monitored,
3062 (uap->pages_reclaimed) ? &pages_reclaimed : NULL,
3063 &pages_wanted);
3064
3065 switch (kr) {
3066 case KERN_SUCCESS:
3067 break;
3068 case KERN_ABORTED:
3069 return EINTR;
3070 default:
3071 return EINVAL;
3072 }
3073
3074 if (uap->pages_reclaimed) {
3075 if (copyout((void *)&pages_reclaimed,
3076 uap->pages_reclaimed,
3077 sizeof(pages_reclaimed)) != 0) {
3078 return EFAULT;
3079 }
3080 }
3081
3082 *retval = (int) pages_wanted;
3083 return 0;
3084 }
3085
3086 int
3087 kas_info(struct proc *p,
3088 struct kas_info_args *uap,
3089 int *retval __unused)
3090 {
3091 #ifndef CONFIG_KAS_INFO
3092 (void)p;
3093 (void)uap;
3094 return ENOTSUP;
3095 #else /* CONFIG_KAS_INFO */
3096 int selector = uap->selector;
3097 user_addr_t valuep = uap->value;
3098 user_addr_t sizep = uap->size;
3099 user_size_t size, rsize;
3100 int error;
3101
3102 if (!kauth_cred_issuser(kauth_cred_get())) {
3103 return EPERM;
3104 }
3105
3106 #if CONFIG_MACF
3107 error = mac_system_check_kas_info(kauth_cred_get(), selector);
3108 if (error) {
3109 return error;
3110 }
3111 #endif
3112
3113 if (IS_64BIT_PROCESS(p)) {
3114 user64_size_t size64;
3115 error = copyin(sizep, &size64, sizeof(size64));
3116 size = (user_size_t)size64;
3117 } else {
3118 user32_size_t size32;
3119 error = copyin(sizep, &size32, sizeof(size32));
3120 size = (user_size_t)size32;
3121 }
3122 if (error) {
3123 return error;
3124 }
3125
3126 switch (selector) {
3127 case KAS_INFO_KERNEL_TEXT_SLIDE_SELECTOR:
3128 {
3129 uint64_t slide = vm_kernel_slide;
3130
3131 if (sizeof(slide) != size) {
3132 return EINVAL;
3133 }
3134
3135 error = copyout(&slide, valuep, sizeof(slide));
3136 if (error) {
3137 return error;
3138 }
3139 rsize = size;
3140 }
3141 break;
3142 case KAS_INFO_KERNEL_SEGMENT_VMADDR_SELECTOR:
3143 {
3144 uint32_t i;
3145 kernel_mach_header_t *mh = &_mh_execute_header;
3146 struct load_command *cmd;
3147 cmd = (struct load_command*) &mh[1];
3148 uint64_t *bases;
3149 rsize = mh->ncmds * sizeof(uint64_t);
3150
3151 /*
3152 * Return the size if no data was passed
3153 */
3154 if (valuep == 0) {
3155 break;
3156 }
3157
3158 if (rsize > size) {
3159 return EINVAL;
3160 }
3161
3162 bases = kheap_alloc(KHEAP_TEMP, rsize, Z_WAITOK | Z_ZERO);
3163
3164 for (i = 0; i < mh->ncmds; i++) {
3165 if (cmd->cmd == LC_SEGMENT_KERNEL) {
3166 __IGNORE_WCASTALIGN(kernel_segment_command_t * sg = (kernel_segment_command_t *) cmd);
3167 bases[i] = (uint64_t)sg->vmaddr;
3168 }
3169 cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
3170 }
3171
3172 error = copyout(bases, valuep, rsize);
3173
3174 kheap_free(KHEAP_TEMP, bases, rsize);
3175
3176 if (error) {
3177 return error;
3178 }
3179 }
3180 break;
3181 default:
3182 return EINVAL;
3183 }
3184
3185 if (IS_64BIT_PROCESS(p)) {
3186 user64_size_t size64 = (user64_size_t)rsize;
3187 error = copyout(&size64, sizep, sizeof(size64));
3188 } else {
3189 user32_size_t size32 = (user32_size_t)rsize;
3190 error = copyout(&size32, sizep, sizeof(size32));
3191 }
3192
3193 return error;
3194 #endif /* CONFIG_KAS_INFO */
3195 }
3196
3197 #if __has_feature(ptrauth_calls)
3198 /*
3199 * Generate a random pointer signing key that isn't 0.
3200 */
3201 uint64_t
3202 generate_jop_key(void)
3203 {
3204 uint64_t key;
3205
3206 do {
3207 read_random(&key, sizeof key);
3208 } while (key == 0);
3209 return key;
3210 }
3211 #endif /* __has_feature(ptrauth_calls) */
3212
3213
3214 #pragma clang diagnostic push
3215 #pragma clang diagnostic ignored "-Wcast-qual"
3216 #pragma clang diagnostic ignored "-Wunused-function"
3217
3218 static void
3219 asserts()
3220 {
3221 static_assert(sizeof(vm_min_kernel_address) == sizeof(unsigned long));
3222 static_assert(sizeof(vm_max_kernel_address) == sizeof(unsigned long));
3223 }
3224
3225 SYSCTL_ULONG(_vm, OID_AUTO, vm_min_kernel_address, CTLFLAG_RD, (unsigned long *) &vm_min_kernel_address, "");
3226 SYSCTL_ULONG(_vm, OID_AUTO, vm_max_kernel_address, CTLFLAG_RD, (unsigned long *) &vm_max_kernel_address, "");
3227 #pragma clang diagnostic pop
3228
3229 extern uint32_t vm_page_pages;
3230 SYSCTL_UINT(_vm, OID_AUTO, pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pages, 0, "");
3231
3232 extern uint32_t vm_page_busy_absent_skipped;
3233 SYSCTL_UINT(_vm, OID_AUTO, page_busy_absent_skipped, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_busy_absent_skipped, 0, "");
3234
3235 extern uint32_t vm_page_upl_tainted;
3236 SYSCTL_UINT(_vm, OID_AUTO, upl_pages_tainted, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_upl_tainted, 0, "");
3237
3238 extern uint32_t vm_page_iopl_tainted;
3239 SYSCTL_UINT(_vm, OID_AUTO, iopl_pages_tainted, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_iopl_tainted, 0, "");
3240
3241 #if (__arm__ || __arm64__) && (DEVELOPMENT || DEBUG)
3242 extern int vm_footprint_suspend_allowed;
3243 SYSCTL_INT(_vm, OID_AUTO, footprint_suspend_allowed, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_footprint_suspend_allowed, 0, "");
3244
3245 extern void pmap_footprint_suspend(vm_map_t map, boolean_t suspend);
3246 static int
3247 sysctl_vm_footprint_suspend SYSCTL_HANDLER_ARGS
3248 {
3249 #pragma unused(oidp, arg1, arg2)
3250 int error = 0;
3251 int new_value;
3252
3253 if (req->newptr == USER_ADDR_NULL) {
3254 return 0;
3255 }
3256 error = SYSCTL_IN(req, &new_value, sizeof(int));
3257 if (error) {
3258 return error;
3259 }
3260 if (!vm_footprint_suspend_allowed) {
3261 if (new_value != 0) {
3262 /* suspends are not allowed... */
3263 return 0;
3264 }
3265 /* ... but let resumes proceed */
3266 }
3267 DTRACE_VM2(footprint_suspend,
3268 vm_map_t, current_map(),
3269 int, new_value);
3270
3271 pmap_footprint_suspend(current_map(), new_value);
3272
3273 return 0;
3274 }
3275 SYSCTL_PROC(_vm, OID_AUTO, footprint_suspend,
3276 CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_LOCKED | CTLFLAG_MASKED,
3277 0, 0, &sysctl_vm_footprint_suspend, "I", "");
3278 #endif /* (__arm__ || __arm64__) && (DEVELOPMENT || DEBUG) */
3279
3280 extern uint64_t vm_map_corpse_footprint_count;
3281 extern uint64_t vm_map_corpse_footprint_size_avg;
3282 extern uint64_t vm_map_corpse_footprint_size_max;
3283 extern uint64_t vm_map_corpse_footprint_full;
3284 extern uint64_t vm_map_corpse_footprint_no_buf;
3285 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_count,
3286 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_count, "");
3287 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_size_avg,
3288 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_size_avg, "");
3289 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_size_max,
3290 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_size_max, "");
3291 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_full,
3292 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_full, "");
3293 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_no_buf,
3294 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_no_buf, "");
3295
3296 #if PMAP_CS
3297 extern uint64_t vm_cs_defer_to_pmap_cs;
3298 extern uint64_t vm_cs_defer_to_pmap_cs_not;
3299 SYSCTL_QUAD(_vm, OID_AUTO, cs_defer_to_pmap_cs,
3300 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_cs_defer_to_pmap_cs, "");
3301 SYSCTL_QUAD(_vm, OID_AUTO, cs_defer_to_pmap_cs_not,
3302 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_cs_defer_to_pmap_cs_not, "");
3303 #endif /* PMAP_CS */
3304
3305 extern uint64_t shared_region_pager_copied;
3306 extern uint64_t shared_region_pager_slid;
3307 extern uint64_t shared_region_pager_slid_error;
3308 extern uint64_t shared_region_pager_reclaimed;
3309 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_copied,
3310 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_copied, "");
3311 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_slid,
3312 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_slid, "");
3313 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_slid_error,
3314 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_slid_error, "");
3315 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_reclaimed,
3316 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_reclaimed, "");
3317 extern int shared_region_destroy_delay;
3318 SYSCTL_INT(_vm, OID_AUTO, shared_region_destroy_delay,
3319 CTLFLAG_RW | CTLFLAG_LOCKED, &shared_region_destroy_delay, 0, "");
3320
3321 #if MACH_ASSERT
3322 extern int pmap_ledgers_panic_leeway;
3323 SYSCTL_INT(_vm, OID_AUTO, pmap_ledgers_panic_leeway, CTLFLAG_RW | CTLFLAG_LOCKED, &pmap_ledgers_panic_leeway, 0, "");
3324 #endif /* MACH_ASSERT */
3325
3326 extern int vm_protect_privileged_from_untrusted;
3327 SYSCTL_INT(_vm, OID_AUTO, protect_privileged_from_untrusted,
3328 CTLFLAG_RW | CTLFLAG_LOCKED, &vm_protect_privileged_from_untrusted, 0, "");
3329 extern uint64_t vm_copied_on_read;
3330 SYSCTL_QUAD(_vm, OID_AUTO, copied_on_read,
3331 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_copied_on_read, "");
3332
3333 extern int vm_shared_region_count;
3334 extern int vm_shared_region_peak;
3335 SYSCTL_INT(_vm, OID_AUTO, shared_region_count,
3336 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_shared_region_count, 0, "");
3337 SYSCTL_INT(_vm, OID_AUTO, shared_region_peak,
3338 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_shared_region_peak, 0, "");
3339 #if DEVELOPMENT || DEBUG
3340 extern unsigned int shared_region_pagers_resident_count;
3341 SYSCTL_INT(_vm, OID_AUTO, shared_region_pagers_resident_count,
3342 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pagers_resident_count, 0, "");
3343 extern unsigned int shared_region_pagers_resident_peak;
3344 SYSCTL_INT(_vm, OID_AUTO, shared_region_pagers_resident_peak,
3345 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pagers_resident_peak, 0, "");
3346 extern int shared_region_pager_count;
3347 SYSCTL_INT(_vm, OID_AUTO, shared_region_pager_count,
3348 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_count, 0, "");
3349 #if __has_feature(ptrauth_calls)
3350 extern int shared_region_key_count;
3351 SYSCTL_INT(_vm, OID_AUTO, shared_region_key_count,
3352 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_key_count, 0, "");
3353 extern int vm_shared_region_reslide_count;
3354 SYSCTL_INT(_vm, OID_AUTO, shared_region_reslide_count,
3355 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_shared_region_reslide_count, 0, "");
3356 #endif /* __has_feature(ptrauth_calls) */
3357 #endif /* DEVELOPMENT || DEBUG */
3358
3359 #if MACH_ASSERT
3360 extern int debug4k_filter;
3361 SYSCTL_INT(_vm, OID_AUTO, debug4k_filter, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_filter, 0, "");
3362 extern int debug4k_panic_on_terminate;
3363 SYSCTL_INT(_vm, OID_AUTO, debug4k_panic_on_terminate, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_panic_on_terminate, 0, "");
3364 extern int debug4k_panic_on_exception;
3365 SYSCTL_INT(_vm, OID_AUTO, debug4k_panic_on_exception, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_panic_on_exception, 0, "");
3366 extern int debug4k_panic_on_misaligned_sharing;
3367 SYSCTL_INT(_vm, OID_AUTO, debug4k_panic_on_misaligned_sharing, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_panic_on_misaligned_sharing, 0, "");
3368 #endif /* MACH_ASSERT */
3369
3370 /*
3371 * A sysctl which causes all existing shared regions to become stale. They
3372 * will no longer be used by anything new and will be torn down as soon as
3373 * the last existing user exits. A write of non-zero value causes that to happen.
3374 * This should only be used by launchd, so we check that this is initproc.
3375 */
3376 static int
3377 shared_region_pivot(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
3378 {
3379 unsigned int value = 0;
3380 int changed = 0;
3381 int error = sysctl_io_number(req, 0, sizeof(value), &value, &changed);
3382 if (error || !changed) {
3383 return error;
3384 }
3385 if (current_proc() != initproc) {
3386 return EPERM;
3387 }
3388
3389 vm_shared_region_pivot();
3390
3391 return 0;
3392 }
3393
3394 SYSCTL_PROC(_vm, OID_AUTO, shared_region_pivot,
3395 CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED,
3396 0, 0, shared_region_pivot, "I", "");
3397
3398 extern int vm_remap_old_path, vm_remap_new_path;
3399 SYSCTL_INT(_vm, OID_AUTO, remap_old_path,
3400 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_remap_old_path, 0, "");
3401 SYSCTL_INT(_vm, OID_AUTO, remap_new_path,
3402 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_remap_new_path, 0, "");
mirror of XNU