]> git.saurik.com Git - apple/xnu.git/blob - bsd/vm/vm_unix.c
projects / apple / xnu.git / blob
? search:


67aceb8d18ec4c589575a01831e590e8dc4fb731
[apple/xnu.git] / bsd / vm / vm_unix.c
1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Mach Operating System
30 * Copyright (c) 1987 Carnegie-Mellon University
31 * All rights reserved. The CMU software License Agreement specifies
32 * the terms and conditions for use and redistribution.
33 */
34 /*
35 * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
36 * support for mandatory and extensible security protections. This notice
37 * is included in support of clause 2.2 (b) of the Apple Public License,
38 * Version 2.0.
39 */
40 #include <vm/vm_options.h>
41
42 #include <kern/task.h>
43 #include <kern/thread.h>
44 #include <kern/debug.h>
45 #include <kern/extmod_statistics.h>
46 #include <mach/mach_traps.h>
47 #include <mach/port.h>
48 #include <mach/sdt.h>
49 #include <mach/task.h>
50 #include <mach/task_access.h>
51 #include <mach/task_special_ports.h>
52 #include <mach/time_value.h>
53 #include <mach/vm_map.h>
54 #include <mach/vm_param.h>
55 #include <mach/vm_prot.h>
56 #include <machine/machine_routines.h>
57
58 #include <sys/file_internal.h>
59 #include <sys/param.h>
60 #include <sys/systm.h>
61 #include <sys/dir.h>
62 #include <sys/namei.h>
63 #include <sys/proc_internal.h>
64 #include <sys/kauth.h>
65 #include <sys/vm.h>
66 #include <sys/file.h>
67 #include <sys/vnode_internal.h>
68 #include <sys/mount.h>
69 #include <sys/xattr.h>
70 #include <sys/trace.h>
71 #include <sys/kernel.h>
72 #include <sys/ubc_internal.h>
73 #include <sys/user.h>
74 #include <sys/syslog.h>
75 #include <sys/stat.h>
76 #include <sys/sysproto.h>
77 #include <sys/mman.h>
78 #include <sys/sysctl.h>
79 #include <sys/cprotect.h>
80 #include <sys/kpi_socket.h>
81 #include <sys/kas_info.h>
82 #include <sys/socket.h>
83 #include <sys/socketvar.h>
84 #include <sys/random.h>
85 #if NECP
86 #include <net/necp.h>
87 #endif /* NECP */
88
89 #include <security/audit/audit.h>
90 #include <security/mac.h>
91 #include <bsm/audit_kevents.h>
92
93 #include <kern/kalloc.h>
94 #include <vm/vm_map.h>
95 #include <vm/vm_kern.h>
96 #include <vm/vm_pageout.h>
97
98 #include <mach/shared_region.h>
99 #include <vm/vm_shared_region.h>
100
101 #include <vm/vm_protos.h>
102
103 #include <sys/kern_memorystatus.h>
104 #include <sys/kern_memorystatus_freeze.h>
105 #include <sys/proc_internal.h>
106
107 #if CONFIG_MACF
108 #include <security/mac_framework.h>
109 #endif
110
111 #include <kern/bits.h>
112
113 #if CONFIG_CSR
114 #include <sys/csr.h>
115 #endif /* CONFIG_CSR */
116 #include <IOKit/IOBSD.h>
117
118 #if VM_MAP_DEBUG_APPLE_PROTECT
119 SYSCTL_INT(_vm, OID_AUTO, map_debug_apple_protect, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_map_debug_apple_protect, 0, "");
120 #endif /* VM_MAP_DEBUG_APPLE_PROTECT */
121
122 #if VM_MAP_DEBUG_FOURK
123 SYSCTL_INT(_vm, OID_AUTO, map_debug_fourk, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_map_debug_fourk, 0, "");
124 #endif /* VM_MAP_DEBUG_FOURK */
125
126 #if DEVELOPMENT || DEBUG
127
128 static int
129 sysctl_kmem_alloc_contig SYSCTL_HANDLER_ARGS
130 {
131 #pragma unused(arg1, arg2)
132 vm_offset_t kaddr;
133 kern_return_t kr;
134 int error = 0;
135 int size = 0;
136
137 error = sysctl_handle_int(oidp, &size, 0, req);
138 if (error || !req->newptr) {
139 return error;
140 }
141
142 kr = kmem_alloc_contig(kernel_map, &kaddr, (vm_size_t)size, 0, 0, 0, 0, VM_KERN_MEMORY_IOKIT);
143
144 if (kr == KERN_SUCCESS) {
145 kmem_free(kernel_map, kaddr, size);
146 }
147
148 return error;
149 }
150
151 SYSCTL_PROC(_vm, OID_AUTO, kmem_alloc_contig, CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED,
152 0, 0, &sysctl_kmem_alloc_contig, "I", "");
153
154 extern int vm_region_footprint;
155 SYSCTL_INT(_vm, OID_AUTO, region_footprint, CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_LOCKED, &vm_region_footprint, 0, "");
156
157 #endif /* DEVELOPMENT || DEBUG */
158
159 static int
160 sysctl_vm_self_region_footprint SYSCTL_HANDLER_ARGS
161 {
162 #pragma unused(arg1, arg2, oidp)
163 int error = 0;
164 int value;
165
166 value = task_self_region_footprint();
167 error = SYSCTL_OUT(req, &value, sizeof(int));
168 if (error) {
169 return error;
170 }
171
172 if (!req->newptr) {
173 return 0;
174 }
175
176 error = SYSCTL_IN(req, &value, sizeof(int));
177 if (error) {
178 return error;
179 }
180 task_self_region_footprint_set(value);
181 return 0;
182 }
183 SYSCTL_PROC(_vm, OID_AUTO, self_region_footprint, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_LOCKED | CTLFLAG_MASKED, 0, 0, &sysctl_vm_self_region_footprint, "I", "");
184
185 static int
186 sysctl_vm_self_region_page_size SYSCTL_HANDLER_ARGS
187 {
188 #pragma unused(arg1, arg2, oidp)
189 int error = 0;
190 int value;
191
192 value = (1 << thread_self_region_page_shift());
193 error = SYSCTL_OUT(req, &value, sizeof(int));
194 if (error) {
195 return error;
196 }
197
198 if (!req->newptr) {
199 return 0;
200 }
201
202 error = SYSCTL_IN(req, &value, sizeof(int));
203 if (error) {
204 return error;
205 }
206
207 if (value != 0 && value != 4096 && value != 16384) {
208 return EINVAL;
209 }
210
211 #if !__ARM_MIXED_PAGE_SIZE__
212 if (value != vm_map_page_size(current_map())) {
213 return EINVAL;
214 }
215 #endif /* !__ARM_MIXED_PAGE_SIZE__ */
216
217 thread_self_region_page_shift_set(bit_first(value));
218 return 0;
219 }
220 SYSCTL_PROC(_vm, OID_AUTO, self_region_page_size, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY | CTLFLAG_LOCKED | CTLFLAG_MASKED, 0, 0, &sysctl_vm_self_region_page_size, "I", "");
221
222
223 #if DEVELOPMENT || DEBUG
224 extern int panic_on_unsigned_execute;
225 SYSCTL_INT(_vm, OID_AUTO, panic_on_unsigned_execute, CTLFLAG_RW | CTLFLAG_LOCKED, &panic_on_unsigned_execute, 0, "");
226 #endif /* DEVELOPMENT || DEBUG */
227
228 extern int cs_executable_create_upl;
229 extern int cs_executable_wire;
230 SYSCTL_INT(_vm, OID_AUTO, cs_executable_create_upl, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_create_upl, 0, "");
231 SYSCTL_INT(_vm, OID_AUTO, cs_executable_wire, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_executable_wire, 0, "");
232
233 extern int apple_protect_pager_count;
234 extern int apple_protect_pager_count_mapped;
235 extern unsigned int apple_protect_pager_cache_limit;
236 SYSCTL_INT(_vm, OID_AUTO, apple_protect_pager_count, CTLFLAG_RD | CTLFLAG_LOCKED, &apple_protect_pager_count, 0, "");
237 SYSCTL_INT(_vm, OID_AUTO, apple_protect_pager_count_mapped, CTLFLAG_RD | CTLFLAG_LOCKED, &apple_protect_pager_count_mapped, 0, "");
238 SYSCTL_UINT(_vm, OID_AUTO, apple_protect_pager_cache_limit, CTLFLAG_RW | CTLFLAG_LOCKED, &apple_protect_pager_cache_limit, 0, "");
239
240 #if DEVELOPMENT || DEBUG
241 extern int radar_20146450;
242 SYSCTL_INT(_vm, OID_AUTO, radar_20146450, CTLFLAG_RW | CTLFLAG_LOCKED, &radar_20146450, 0, "");
243
244 extern int macho_printf;
245 SYSCTL_INT(_vm, OID_AUTO, macho_printf, CTLFLAG_RW | CTLFLAG_LOCKED, &macho_printf, 0, "");
246
247 extern int apple_protect_pager_data_request_debug;
248 SYSCTL_INT(_vm, OID_AUTO, apple_protect_pager_data_request_debug, CTLFLAG_RW | CTLFLAG_LOCKED, &apple_protect_pager_data_request_debug, 0, "");
249
250 #if __arm__ || __arm64__
251 /* These are meant to support the page table accounting unit test. */
252 extern unsigned int arm_hardware_page_size;
253 extern unsigned int arm_pt_desc_size;
254 extern unsigned int arm_pt_root_size;
255 extern unsigned int free_page_size_tt_count;
256 extern unsigned int free_two_page_size_tt_count;
257 extern unsigned int free_tt_count;
258 extern unsigned int inuse_user_tteroot_count;
259 extern unsigned int inuse_kernel_tteroot_count;
260 extern unsigned int inuse_user_ttepages_count;
261 extern unsigned int inuse_kernel_ttepages_count;
262 extern unsigned int inuse_user_ptepages_count;
263 extern unsigned int inuse_kernel_ptepages_count;
264 SYSCTL_UINT(_vm, OID_AUTO, native_hw_pagesize, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_hardware_page_size, 0, "");
265 SYSCTL_UINT(_vm, OID_AUTO, arm_pt_desc_size, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_pt_desc_size, 0, "");
266 SYSCTL_UINT(_vm, OID_AUTO, arm_pt_root_size, CTLFLAG_RD | CTLFLAG_LOCKED, &arm_pt_root_size, 0, "");
267 SYSCTL_UINT(_vm, OID_AUTO, free_1page_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_page_size_tt_count, 0, "");
268 SYSCTL_UINT(_vm, OID_AUTO, free_2page_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_two_page_size_tt_count, 0, "");
269 SYSCTL_UINT(_vm, OID_AUTO, free_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &free_tt_count, 0, "");
270 SYSCTL_UINT(_vm, OID_AUTO, user_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_tteroot_count, 0, "");
271 SYSCTL_UINT(_vm, OID_AUTO, kernel_tte_root, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_tteroot_count, 0, "");
272 SYSCTL_UINT(_vm, OID_AUTO, user_tte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_ttepages_count, 0, "");
273 SYSCTL_UINT(_vm, OID_AUTO, kernel_tte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_ttepages_count, 0, "");
274 SYSCTL_UINT(_vm, OID_AUTO, user_pte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_user_ptepages_count, 0, "");
275 SYSCTL_UINT(_vm, OID_AUTO, kernel_pte_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &inuse_kernel_ptepages_count, 0, "");
276 #if DEVELOPMENT || DEBUG
277 extern unsigned long pmap_asid_flushes;
278 SYSCTL_ULONG(_vm, OID_AUTO, pmap_asid_flushes, CTLFLAG_RD | CTLFLAG_LOCKED, &pmap_asid_flushes, "");
279 extern unsigned long pmap_asid_hits;
280 SYSCTL_ULONG(_vm, OID_AUTO, pmap_asid_hits, CTLFLAG_RD | CTLFLAG_LOCKED, &pmap_asid_hits, "");
281 extern unsigned long pmap_asid_misses;
282 SYSCTL_ULONG(_vm, OID_AUTO, pmap_asid_misses, CTLFLAG_RD | CTLFLAG_LOCKED, &pmap_asid_misses, "");
283 #endif
284 #endif /* __arm__ || __arm64__ */
285
286 #if __arm64__
287 extern int fourk_pager_data_request_debug;
288 SYSCTL_INT(_vm, OID_AUTO, fourk_pager_data_request_debug, CTLFLAG_RW | CTLFLAG_LOCKED, &fourk_pager_data_request_debug, 0, "");
289 #endif /* __arm64__ */
290 #endif /* DEVELOPMENT || DEBUG */
291
292 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_compressor, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_compressor, 0, "");
293 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_compressor_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_compressor_pages, 0, "");
294 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_terminate, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_terminate, 0, "");
295 SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_terminate_failure, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_terminate_failure, 0, "");
296 SYSCTL_INT(_vm, OID_AUTO, vm_should_cow_but_wired, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.should_cow_but_wired, 0, "");
297 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_extra_cow, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_extra_cow, 0, "");
298 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_extra_cow_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_extra_cow_pages, 0, "");
299 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_lookup_failure_write, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_lookup_failure_write, 0, "");
300 SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_lookup_failure_copy, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_lookup_failure_copy, 0, "");
301 #if VM_SCAN_FOR_SHADOW_CHAIN
302 static int vm_shadow_max_enabled = 0; /* Disabled by default */
303 extern int proc_shadow_max(void);
304 static int
305 vm_shadow_max SYSCTL_HANDLER_ARGS
306 {
307 #pragma unused(arg1, arg2, oidp)
308 int value = 0;
309
310 if (vm_shadow_max_enabled) {
311 value = proc_shadow_max();
312 }
313
314 return SYSCTL_OUT(req, &value, sizeof(value));
315 }
316 SYSCTL_PROC(_vm, OID_AUTO, vm_shadow_max, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
317 0, 0, &vm_shadow_max, "I", "");
318
319 SYSCTL_INT(_vm, OID_AUTO, vm_shadow_max_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_shadow_max_enabled, 0, "");
320
321 #endif /* VM_SCAN_FOR_SHADOW_CHAIN */
322
323 SYSCTL_INT(_vm, OID_AUTO, vm_debug_events, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_debug_events, 0, "");
324
325 __attribute__((noinline)) int __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(
326 mach_port_t task_access_port, int32_t calling_pid, uint32_t calling_gid, int32_t target_pid, mach_task_flavor_t flavor);
327 /*
328 * Sysctl's related to data/stack execution. See osfmk/vm/vm_map.c
329 */
330
331 #if DEVELOPMENT || DEBUG
332 extern int allow_stack_exec, allow_data_exec;
333
334 SYSCTL_INT(_vm, OID_AUTO, allow_stack_exec, CTLFLAG_RW | CTLFLAG_LOCKED, &allow_stack_exec, 0, "");
335 SYSCTL_INT(_vm, OID_AUTO, allow_data_exec, CTLFLAG_RW | CTLFLAG_LOCKED, &allow_data_exec, 0, "");
336
337 #endif /* DEVELOPMENT || DEBUG */
338
339 static const char *prot_values[] = {
340 "none",
341 "read-only",
342 "write-only",
343 "read-write",
344 "execute-only",
345 "read-execute",
346 "write-execute",
347 "read-write-execute"
348 };
349
350 void
351 log_stack_execution_failure(addr64_t vaddr, vm_prot_t prot)
352 {
353 printf("Data/Stack execution not permitted: %s[pid %d] at virtual address 0x%qx, protections were %s\n",
354 current_proc()->p_comm, current_proc()->p_pid, vaddr, prot_values[prot & VM_PROT_ALL]);
355 }
356
357 /*
358 * shared_region_unnest_logging: level of logging of unnesting events
359 * 0 - no logging
360 * 1 - throttled logging of unexpected unnesting events (default)
361 * 2 - unthrottled logging of unexpected unnesting events
362 * 3+ - unthrottled logging of all unnesting events
363 */
364 int shared_region_unnest_logging = 1;
365
366 SYSCTL_INT(_vm, OID_AUTO, shared_region_unnest_logging, CTLFLAG_RW | CTLFLAG_LOCKED,
367 &shared_region_unnest_logging, 0, "");
368
369 int vm_shared_region_unnest_log_interval = 10;
370 int shared_region_unnest_log_count_threshold = 5;
371
372 /*
373 * Shared cache path enforcement.
374 */
375
376 #if XNU_TARGET_OS_OSX
377
378 #if defined (__x86_64__)
379 static int scdir_enforce = 1;
380 #else /* defined (__x86_64__) */
381 static int scdir_enforce = 0; /* AOT caches live elsewhere */
382 #endif /* defined (__x86_64__) */
383
384 static char scdir_path[] = "/System/Library/dyld/";
385
386 #else /* XNU_TARGET_OS_OSX */
387
388 static int scdir_enforce = 0;
389 static char scdir_path[] = "/System/Library/Caches/com.apple.dyld/";
390
391 #endif /* XNU_TARGET_OS_OSX */
392
393 #ifndef SECURE_KERNEL
394 static int sysctl_scdir_enforce SYSCTL_HANDLER_ARGS
395 {
396 #if CONFIG_CSR
397 if (csr_check(CSR_ALLOW_UNRESTRICTED_FS) != 0) {
398 printf("Failed attempt to set vm.enforce_shared_cache_dir sysctl\n");
399 return EPERM;
400 }
401 #endif /* CONFIG_CSR */
402 return sysctl_handle_int(oidp, arg1, arg2, req);
403 }
404
405 SYSCTL_PROC(_vm, OID_AUTO, enforce_shared_cache_dir, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &scdir_enforce, 0, sysctl_scdir_enforce, "I", "");
406 #endif
407
408 /* These log rate throttling state variables aren't thread safe, but
409 * are sufficient unto the task.
410 */
411 static int64_t last_unnest_log_time = 0;
412 static int shared_region_unnest_log_count = 0;
413
414 void
415 log_unnest_badness(
416 vm_map_t m,
417 vm_map_offset_t s,
418 vm_map_offset_t e,
419 boolean_t is_nested_map,
420 vm_map_offset_t lowest_unnestable_addr)
421 {
422 struct timeval tv;
423
424 if (shared_region_unnest_logging == 0) {
425 return;
426 }
427
428 if (shared_region_unnest_logging <= 2 &&
429 is_nested_map &&
430 s >= lowest_unnestable_addr) {
431 /*
432 * Unnesting of writable map entries is fine.
433 */
434 return;
435 }
436
437 if (shared_region_unnest_logging <= 1) {
438 microtime(&tv);
439 if ((tv.tv_sec - last_unnest_log_time) <
440 vm_shared_region_unnest_log_interval) {
441 if (shared_region_unnest_log_count++ >
442 shared_region_unnest_log_count_threshold) {
443 return;
444 }
445 } else {
446 last_unnest_log_time = tv.tv_sec;
447 shared_region_unnest_log_count = 0;
448 }
449 }
450
451 DTRACE_VM4(log_unnest_badness,
452 vm_map_t, m,
453 vm_map_offset_t, s,
454 vm_map_offset_t, e,
455 vm_map_offset_t, lowest_unnestable_addr);
456 printf("%s[%d] triggered unnest of range 0x%qx->0x%qx of DYLD shared region in VM map %p. While not abnormal for debuggers, this increases system memory footprint until the target exits.\n", current_proc()->p_comm, current_proc()->p_pid, (uint64_t)s, (uint64_t)e, (void *) VM_KERNEL_ADDRPERM(m));
457 }
458
459 int
460 useracc(
461 user_addr_t addr,
462 user_size_t len,
463 int prot)
464 {
465 vm_map_t map;
466
467 map = current_map();
468 return vm_map_check_protection(
469 map,
470 vm_map_trunc_page(addr,
471 vm_map_page_mask(map)),
472 vm_map_round_page(addr + len,
473 vm_map_page_mask(map)),
474 prot == B_READ ? VM_PROT_READ : VM_PROT_WRITE);
475 }
476
477 int
478 vslock(
479 user_addr_t addr,
480 user_size_t len)
481 {
482 kern_return_t kret;
483 vm_map_t map;
484
485 map = current_map();
486 kret = vm_map_wire_kernel(map,
487 vm_map_trunc_page(addr,
488 vm_map_page_mask(map)),
489 vm_map_round_page(addr + len,
490 vm_map_page_mask(map)),
491 VM_PROT_READ | VM_PROT_WRITE, VM_KERN_MEMORY_BSD,
492 FALSE);
493
494 switch (kret) {
495 case KERN_SUCCESS:
496 return 0;
497 case KERN_INVALID_ADDRESS:
498 case KERN_NO_SPACE:
499 return ENOMEM;
500 case KERN_PROTECTION_FAILURE:
501 return EACCES;
502 default:
503 return EINVAL;
504 }
505 }
506
507 int
508 vsunlock(
509 user_addr_t addr,
510 user_size_t len,
511 __unused int dirtied)
512 {
513 #if FIXME /* [ */
514 pmap_t pmap;
515 vm_page_t pg;
516 vm_map_offset_t vaddr;
517 ppnum_t paddr;
518 #endif /* FIXME ] */
519 kern_return_t kret;
520 vm_map_t map;
521
522 map = current_map();
523
524 #if FIXME /* [ */
525 if (dirtied) {
526 pmap = get_task_pmap(current_task());
527 for (vaddr = vm_map_trunc_page(addr, PAGE_MASK);
528 vaddr < vm_map_round_page(addr + len, PAGE_MASK);
529 vaddr += PAGE_SIZE) {
530 paddr = pmap_find_phys(pmap, vaddr);
531 pg = PHYS_TO_VM_PAGE(paddr);
532 vm_page_set_modified(pg);
533 }
534 }
535 #endif /* FIXME ] */
536 #ifdef lint
537 dirtied++;
538 #endif /* lint */
539 kret = vm_map_unwire(map,
540 vm_map_trunc_page(addr,
541 vm_map_page_mask(map)),
542 vm_map_round_page(addr + len,
543 vm_map_page_mask(map)),
544 FALSE);
545 switch (kret) {
546 case KERN_SUCCESS:
547 return 0;
548 case KERN_INVALID_ADDRESS:
549 case KERN_NO_SPACE:
550 return ENOMEM;
551 case KERN_PROTECTION_FAILURE:
552 return EACCES;
553 default:
554 return EINVAL;
555 }
556 }
557
558 int
559 subyte(
560 user_addr_t addr,
561 int byte)
562 {
563 char character;
564
565 character = (char)byte;
566 return copyout((void *)&(character), addr, sizeof(char)) == 0 ? 0 : -1;
567 }
568
569 int
570 suibyte(
571 user_addr_t addr,
572 int byte)
573 {
574 char character;
575
576 character = (char)byte;
577 return copyout((void *)&(character), addr, sizeof(char)) == 0 ? 0 : -1;
578 }
579
580 int
581 fubyte(user_addr_t addr)
582 {
583 unsigned char byte;
584
585 if (copyin(addr, (void *) &byte, sizeof(char))) {
586 return -1;
587 }
588 return byte;
589 }
590
591 int
592 fuibyte(user_addr_t addr)
593 {
594 unsigned char byte;
595
596 if (copyin(addr, (void *) &(byte), sizeof(char))) {
597 return -1;
598 }
599 return byte;
600 }
601
602 int
603 suword(
604 user_addr_t addr,
605 long word)
606 {
607 return copyout((void *) &word, addr, sizeof(int)) == 0 ? 0 : -1;
608 }
609
610 long
611 fuword(user_addr_t addr)
612 {
613 long word = 0;
614
615 if (copyin(addr, (void *) &word, sizeof(int))) {
616 return -1;
617 }
618 return word;
619 }
620
621 /* suiword and fuiword are the same as suword and fuword, respectively */
622
623 int
624 suiword(
625 user_addr_t addr,
626 long word)
627 {
628 return copyout((void *) &word, addr, sizeof(int)) == 0 ? 0 : -1;
629 }
630
631 long
632 fuiword(user_addr_t addr)
633 {
634 long word = 0;
635
636 if (copyin(addr, (void *) &word, sizeof(int))) {
637 return -1;
638 }
639 return word;
640 }
641
642 /*
643 * With a 32-bit kernel and mixed 32/64-bit user tasks, this interface allows the
644 * fetching and setting of process-sized size_t and pointer values.
645 */
646 int
647 sulong(user_addr_t addr, int64_t word)
648 {
649 if (IS_64BIT_PROCESS(current_proc())) {
650 return copyout((void *)&word, addr, sizeof(word)) == 0 ? 0 : -1;
651 } else {
652 return suiword(addr, (long)word);
653 }
654 }
655
656 int64_t
657 fulong(user_addr_t addr)
658 {
659 int64_t longword;
660
661 if (IS_64BIT_PROCESS(current_proc())) {
662 if (copyin(addr, (void *)&longword, sizeof(longword)) != 0) {
663 return -1;
664 }
665 return longword;
666 } else {
667 return (int64_t)fuiword(addr);
668 }
669 }
670
671 int
672 suulong(user_addr_t addr, uint64_t uword)
673 {
674 if (IS_64BIT_PROCESS(current_proc())) {
675 return copyout((void *)&uword, addr, sizeof(uword)) == 0 ? 0 : -1;
676 } else {
677 return suiword(addr, (uint32_t)uword);
678 }
679 }
680
681 uint64_t
682 fuulong(user_addr_t addr)
683 {
684 uint64_t ulongword;
685
686 if (IS_64BIT_PROCESS(current_proc())) {
687 if (copyin(addr, (void *)&ulongword, sizeof(ulongword)) != 0) {
688 return -1ULL;
689 }
690 return ulongword;
691 } else {
692 return (uint64_t)fuiword(addr);
693 }
694 }
695
696 int
697 swapon(__unused proc_t procp, __unused struct swapon_args *uap, __unused int *retval)
698 {
699 return ENOTSUP;
700 }
701
702 /*
703 * pid_for_task
704 *
705 * Find the BSD process ID for the Mach task associated with the given Mach port
706 * name
707 *
708 * Parameters: args User argument descriptor (see below)
709 *
710 * Indirect parameters: args->t Mach port name
711 * args->pid Process ID (returned value; see below)
712 *
713 * Returns: KERL_SUCCESS Success
714 * KERN_FAILURE Not success
715 *
716 * Implicit returns: args->pid Process ID
717 *
718 */
719 kern_return_t
720 pid_for_task(
721 struct pid_for_task_args *args)
722 {
723 mach_port_name_t t = args->t;
724 user_addr_t pid_addr = args->pid;
725 proc_t p;
726 task_t t1;
727 int pid = -1;
728 kern_return_t err = KERN_SUCCESS;
729
730 AUDIT_MACH_SYSCALL_ENTER(AUE_PIDFORTASK);
731 AUDIT_ARG(mach_port1, t);
732
733 t1 = port_name_to_task_name(t);
734
735 if (t1 == TASK_NULL) {
736 err = KERN_FAILURE;
737 goto pftout;
738 } else {
739 p = get_bsdtask_info(t1);
740 if (p) {
741 pid = proc_pid(p);
742 err = KERN_SUCCESS;
743 } else if (is_corpsetask(t1)) {
744 pid = task_pid(t1);
745 err = KERN_SUCCESS;
746 } else {
747 err = KERN_FAILURE;
748 }
749 }
750 task_deallocate(t1);
751 pftout:
752 AUDIT_ARG(pid, pid);
753 (void) copyout((char *) &pid, pid_addr, sizeof(int));
754 AUDIT_MACH_SYSCALL_EXIT(err);
755 return err;
756 }
757
758 /*
759 *
760 * tfp_policy = KERN_TFP_POLICY_DENY; Deny Mode: None allowed except for self
761 * tfp_policy = KERN_TFP_POLICY_DEFAULT; default mode: all posix checks and upcall via task port for authentication
762 *
763 */
764 static int tfp_policy = KERN_TFP_POLICY_DEFAULT;
765
766 /*
767 * Routine: task_for_pid_posix_check
768 * Purpose:
769 * Verify that the current process should be allowed to
770 * get the target process's task port. This is only
771 * permitted if:
772 * - The current process is root
773 * OR all of the following are true:
774 * - The target process's real, effective, and saved uids
775 * are the same as the current proc's euid,
776 * - The target process's group set is a subset of the
777 * calling process's group set, and
778 * - The target process hasn't switched credentials.
779 *
780 * Returns: TRUE: permitted
781 * FALSE: denied
782 */
783 static int
784 task_for_pid_posix_check(proc_t target)
785 {
786 kauth_cred_t targetcred, mycred;
787 uid_t myuid;
788 int allowed;
789
790 /* No task_for_pid on bad targets */
791 if (target->p_stat == SZOMB) {
792 return FALSE;
793 }
794
795 mycred = kauth_cred_get();
796 myuid = kauth_cred_getuid(mycred);
797
798 /* If we're running as root, the check passes */
799 if (kauth_cred_issuser(mycred)) {
800 return TRUE;
801 }
802
803 /* We're allowed to get our own task port */
804 if (target == current_proc()) {
805 return TRUE;
806 }
807
808 /*
809 * Under DENY, only root can get another proc's task port,
810 * so no more checks are needed.
811 */
812 if (tfp_policy == KERN_TFP_POLICY_DENY) {
813 return FALSE;
814 }
815
816 targetcred = kauth_cred_proc_ref(target);
817 allowed = TRUE;
818
819 /* Do target's ruid, euid, and saved uid match my euid? */
820 if ((kauth_cred_getuid(targetcred) != myuid) ||
821 (kauth_cred_getruid(targetcred) != myuid) ||
822 (kauth_cred_getsvuid(targetcred) != myuid)) {
823 allowed = FALSE;
824 goto out;
825 }
826
827 /* Are target's groups a subset of my groups? */
828 if (kauth_cred_gid_subset(targetcred, mycred, &allowed) ||
829 allowed == 0) {
830 allowed = FALSE;
831 goto out;
832 }
833
834 /* Has target switched credentials? */
835 if (target->p_flag & P_SUGID) {
836 allowed = FALSE;
837 goto out;
838 }
839
840 out:
841 kauth_cred_unref(&targetcred);
842 return allowed;
843 }
844
845 /*
846 * __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__
847 *
848 * Description: Waits for the user space daemon to respond to the request
849 * we made. Function declared non inline to be visible in
850 * stackshots and spindumps as well as debugging.
851 */
852 __attribute__((noinline)) int
853 __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(
854 mach_port_t task_access_port, int32_t calling_pid, uint32_t calling_gid, int32_t target_pid, mach_task_flavor_t flavor)
855 {
856 return check_task_access_with_flavor(task_access_port, calling_pid, calling_gid, target_pid, flavor);
857 }
858
859 /*
860 * Routine: task_for_pid
861 * Purpose:
862 * Get the task port for another "process", named by its
863 * process ID on the same host as "target_task".
864 *
865 * Only permitted to privileged processes, or processes
866 * with the same user ID.
867 *
868 * Note: if pid == 0, an error is return no matter who is calling.
869 *
870 * XXX This should be a BSD system call, not a Mach trap!!!
871 */
872 kern_return_t
873 task_for_pid(
874 struct task_for_pid_args *args)
875 {
876 mach_port_name_t target_tport = args->target_tport;
877 int pid = args->pid;
878 user_addr_t task_addr = args->t;
879 proc_t p = PROC_NULL;
880 task_t t1 = TASK_NULL;
881 task_t task = TASK_NULL;
882 mach_port_name_t tret = MACH_PORT_NULL;
883 ipc_port_t tfpport = MACH_PORT_NULL;
884 void * sright = NULL;
885 int error = 0;
886 boolean_t is_current_proc = FALSE;
887 struct proc_ident pident = {0};
888
889 AUDIT_MACH_SYSCALL_ENTER(AUE_TASKFORPID);
890 AUDIT_ARG(pid, pid);
891 AUDIT_ARG(mach_port1, target_tport);
892
893 /* Always check if pid == 0 */
894 if (pid == 0) {
895 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
896 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
897 return KERN_FAILURE;
898 }
899
900 t1 = port_name_to_task(target_tport);
901 if (t1 == TASK_NULL) {
902 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
903 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
904 return KERN_FAILURE;
905 }
906
907
908 p = proc_find(pid);
909 if (p == PROC_NULL) {
910 error = KERN_FAILURE;
911 goto tfpout;
912 }
913 pident = proc_ident(p);
914 is_current_proc = (p == current_proc());
915
916 #if CONFIG_AUDIT
917 AUDIT_ARG(process, p);
918 #endif
919
920 if (!(task_for_pid_posix_check(p))) {
921 error = KERN_FAILURE;
922 goto tfpout;
923 }
924
925 if (p->task == TASK_NULL) {
926 error = KERN_SUCCESS;
927 goto tfpout;
928 }
929
930 /*
931 * Grab a task reference and drop the proc reference as the proc ref
932 * shouldn't be held accross upcalls.
933 */
934 task = p->task;
935 task_reference(task);
936
937 proc_rele(p);
938 p = PROC_NULL;
939
940 #if CONFIG_MACF
941 error = mac_proc_check_get_task(kauth_cred_get(), &pident, TASK_FLAVOR_CONTROL);
942 if (error) {
943 error = KERN_FAILURE;
944 goto tfpout;
945 }
946 #endif
947
948 /* If we aren't root and target's task access port is set... */
949 if (!kauth_cred_issuser(kauth_cred_get()) &&
950 !is_current_proc &&
951 (task_get_task_access_port(task, &tfpport) == 0) &&
952 (tfpport != IPC_PORT_NULL)) {
953 if (tfpport == IPC_PORT_DEAD) {
954 error = KERN_PROTECTION_FAILURE;
955 goto tfpout;
956 }
957
958 /* Call up to the task access server */
959 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport,
960 proc_selfpid(), kauth_getgid(), pid, TASK_FLAVOR_CONTROL);
961
962 if (error != MACH_MSG_SUCCESS) {
963 if (error == MACH_RCV_INTERRUPTED) {
964 error = KERN_ABORTED;
965 } else {
966 error = KERN_FAILURE;
967 }
968 goto tfpout;
969 }
970 }
971
972 /* Grant task port access */
973 extmod_statistics_incr_task_for_pid(task);
974
975 if (task == current_task()) {
976 /* return pinned self if current_task() so equality check with mach_task_self_ passes */
977 sright = (void *)convert_task_to_port_pinned(task);
978 } else {
979 sright = (void *)convert_task_to_port(task);
980 }
981
982 /* Check if the task has been corpsified */
983 if (is_corpsetask(task)) {
984 /* task ref consumed by convert_task_to_port */
985 task = TASK_NULL;
986 ipc_port_release_send(sright);
987 error = KERN_FAILURE;
988 goto tfpout;
989 }
990
991 /* task ref consumed by convert_task_to_port */
992 task = TASK_NULL;
993 tret = ipc_port_copyout_send(
994 sright,
995 get_task_ipcspace(current_task()));
996
997 error = KERN_SUCCESS;
998
999 tfpout:
1000 task_deallocate(t1);
1001 AUDIT_ARG(mach_port2, tret);
1002 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1003
1004 if (tfpport != IPC_PORT_NULL) {
1005 ipc_port_release_send(tfpport);
1006 }
1007 if (task != TASK_NULL) {
1008 task_deallocate(task);
1009 }
1010 if (p != PROC_NULL) {
1011 proc_rele(p);
1012 }
1013 AUDIT_MACH_SYSCALL_EXIT(error);
1014 return error;
1015 }
1016
1017 /*
1018 * Routine: task_name_for_pid
1019 * Purpose:
1020 * Get the task name port for another "process", named by its
1021 * process ID on the same host as "target_task".
1022 *
1023 * Only permitted to privileged processes, or processes
1024 * with the same user ID.
1025 *
1026 * XXX This should be a BSD system call, not a Mach trap!!!
1027 */
1028
1029 kern_return_t
1030 task_name_for_pid(
1031 struct task_name_for_pid_args *args)
1032 {
1033 mach_port_name_t target_tport = args->target_tport;
1034 int pid = args->pid;
1035 user_addr_t task_addr = args->t;
1036 proc_t p = PROC_NULL;
1037 task_t t1 = TASK_NULL;
1038 mach_port_name_t tret = MACH_PORT_NULL;
1039 void * sright;
1040 int error = 0, refheld = 0;
1041 kauth_cred_t target_cred;
1042
1043 AUDIT_MACH_SYSCALL_ENTER(AUE_TASKNAMEFORPID);
1044 AUDIT_ARG(pid, pid);
1045 AUDIT_ARG(mach_port1, target_tport);
1046
1047 t1 = port_name_to_task(target_tport);
1048 if (t1 == TASK_NULL) {
1049 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
1050 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
1051 return KERN_FAILURE;
1052 }
1053
1054 p = proc_find(pid);
1055 if (p != PROC_NULL) {
1056 AUDIT_ARG(process, p);
1057 target_cred = kauth_cred_proc_ref(p);
1058 refheld = 1;
1059
1060 if ((p->p_stat != SZOMB)
1061 && ((current_proc() == p)
1062 || kauth_cred_issuser(kauth_cred_get())
1063 || ((kauth_cred_getuid(target_cred) == kauth_cred_getuid(kauth_cred_get())) &&
1064 ((kauth_cred_getruid(target_cred) == kauth_getruid()))))) {
1065 if (p->task != TASK_NULL) {
1066 struct proc_ident pident = proc_ident(p);
1067
1068 task_t task = p->task;
1069
1070 task_reference(p->task);
1071 proc_rele(p);
1072 p = PROC_NULL;
1073 #if CONFIG_MACF
1074 error = mac_proc_check_get_task(kauth_cred_get(), &pident, TASK_FLAVOR_NAME);
1075 if (error) {
1076 task_deallocate(task);
1077 goto noperm;
1078 }
1079 #endif
1080 sright = (void *)convert_task_name_to_port(task);
1081 task = NULL;
1082 tret = ipc_port_copyout_send(sright,
1083 get_task_ipcspace(current_task()));
1084 } else {
1085 tret = MACH_PORT_NULL;
1086 }
1087
1088 AUDIT_ARG(mach_port2, tret);
1089 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
1090 task_deallocate(t1);
1091 error = KERN_SUCCESS;
1092 goto tnfpout;
1093 }
1094 }
1095
1096 #if CONFIG_MACF
1097 noperm:
1098 #endif
1099 task_deallocate(t1);
1100 tret = MACH_PORT_NULL;
1101 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1102 error = KERN_FAILURE;
1103 tnfpout:
1104 if (refheld != 0) {
1105 kauth_cred_unref(&target_cred);
1106 }
1107 if (p != PROC_NULL) {
1108 proc_rele(p);
1109 }
1110 AUDIT_MACH_SYSCALL_EXIT(error);
1111 return error;
1112 }
1113
1114 /*
1115 * Routine: task_inspect_for_pid
1116 * Purpose:
1117 * Get the task inspect port for another "process", named by its
1118 * process ID on the same host as "target_task".
1119 */
1120 int
1121 task_inspect_for_pid(struct proc *p __unused, struct task_inspect_for_pid_args *args, int *ret)
1122 {
1123 mach_port_name_t target_tport = args->target_tport;
1124 int pid = args->pid;
1125 user_addr_t task_addr = args->t;
1126
1127 proc_t proc = PROC_NULL;
1128 task_t t1 = TASK_NULL;
1129 task_inspect_t task_insp = TASK_INSPECT_NULL;
1130 mach_port_name_t tret = MACH_PORT_NULL;
1131 ipc_port_t tfpport = MACH_PORT_NULL;
1132 int error = 0;
1133 void *sright = NULL;
1134 boolean_t is_current_proc = FALSE;
1135 struct proc_ident pident = {0};
1136
1137 /* Disallow inspect port for kernel_task */
1138 if (pid == 0) {
1139 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
1140 return EPERM;
1141 }
1142
1143 t1 = port_name_to_task(target_tport);
1144 if (t1 == TASK_NULL) {
1145 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1146 return EINVAL;
1147 }
1148
1149 proc = proc_find(pid);
1150 if (proc == PROC_NULL) {
1151 error = ESRCH;
1152 goto tifpout;
1153 }
1154 pident = proc_ident(proc);
1155 is_current_proc = (proc == current_proc());
1156
1157 if (!(task_for_pid_posix_check(proc))) {
1158 error = EPERM;
1159 goto tifpout;
1160 }
1161
1162 task_insp = proc->task;
1163 if (task_insp == TASK_INSPECT_NULL) {
1164 goto tifpout;
1165 }
1166
1167 /*
1168 * Grab a task reference and drop the proc reference before making any upcalls.
1169 */
1170 task_reference(task_insp);
1171
1172 proc_rele(proc);
1173 proc = PROC_NULL;
1174
1175 #if CONFIG_MACF
1176 error = mac_proc_check_get_task(kauth_cred_get(), &pident, TASK_FLAVOR_INSPECT);
1177 if (error) {
1178 error = EPERM;
1179 goto tifpout;
1180 }
1181 #endif
1182
1183 /* If we aren't root and target's task access port is set... */
1184 if (!kauth_cred_issuser(kauth_cred_get()) &&
1185 !is_current_proc &&
1186 (task_get_task_access_port(task_insp, &tfpport) == 0) &&
1187 (tfpport != IPC_PORT_NULL)) {
1188 if (tfpport == IPC_PORT_DEAD) {
1189 error = EACCES;
1190 goto tifpout;
1191 }
1192
1193
1194 /* Call up to the task access server */
1195 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport,
1196 proc_selfpid(), kauth_getgid(), pid, TASK_FLAVOR_INSPECT);
1197
1198 if (error != MACH_MSG_SUCCESS) {
1199 if (error == MACH_RCV_INTERRUPTED) {
1200 error = EINTR;
1201 } else {
1202 error = EPERM;
1203 }
1204 goto tifpout;
1205 }
1206 }
1207
1208 /* Check if the task has been corpsified */
1209 if (is_corpsetask(task_insp)) {
1210 error = EACCES;
1211 goto tifpout;
1212 }
1213
1214 /* could be IP_NULL, consumes a ref */
1215 sright = (void*) convert_task_inspect_to_port(task_insp);
1216 task_insp = TASK_INSPECT_NULL;
1217 tret = ipc_port_copyout_send(sright, get_task_ipcspace(current_task()));
1218
1219 tifpout:
1220 task_deallocate(t1);
1221 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1222 if (proc != PROC_NULL) {
1223 proc_rele(proc);
1224 }
1225 if (tfpport != IPC_PORT_NULL) {
1226 ipc_port_release_send(tfpport);
1227 }
1228 if (task_insp != TASK_INSPECT_NULL) {
1229 task_deallocate(task_insp);
1230 }
1231
1232 *ret = error;
1233 return error;
1234 }
1235
1236 /*
1237 * Routine: task_read_for_pid
1238 * Purpose:
1239 * Get the task read port for another "process", named by its
1240 * process ID on the same host as "target_task".
1241 */
1242 int
1243 task_read_for_pid(struct proc *p __unused, struct task_read_for_pid_args *args, int *ret)
1244 {
1245 mach_port_name_t target_tport = args->target_tport;
1246 int pid = args->pid;
1247 user_addr_t task_addr = args->t;
1248
1249 proc_t proc = PROC_NULL;
1250 task_t t1 = TASK_NULL;
1251 task_read_t task_read = TASK_READ_NULL;
1252 mach_port_name_t tret = MACH_PORT_NULL;
1253 ipc_port_t tfpport = MACH_PORT_NULL;
1254 int error = 0;
1255 void *sright = NULL;
1256 boolean_t is_current_proc = FALSE;
1257 struct proc_ident pident = {0};
1258
1259 /* Disallow read port for kernel_task */
1260 if (pid == 0) {
1261 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
1262 return EPERM;
1263 }
1264
1265 t1 = port_name_to_task(target_tport);
1266 if (t1 == TASK_NULL) {
1267 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
1268 return EINVAL;
1269 }
1270
1271 proc = proc_find(pid);
1272 if (proc == PROC_NULL) {
1273 error = ESRCH;
1274 goto trfpout;
1275 }
1276 pident = proc_ident(proc);
1277 is_current_proc = (proc == current_proc());
1278
1279 if (!(task_for_pid_posix_check(proc))) {
1280 error = EPERM;
1281 goto trfpout;
1282 }
1283
1284 task_read = proc->task;
1285 if (task_read == TASK_INSPECT_NULL) {
1286 goto trfpout;
1287 }
1288
1289 /*
1290 * Grab a task reference and drop the proc reference before making any upcalls.
1291 */
1292 task_reference(task_read);
1293
1294 proc_rele(proc);
1295 proc = PROC_NULL;
1296
1297 #if CONFIG_MACF
1298 error = mac_proc_check_get_task(kauth_cred_get(), &pident, TASK_FLAVOR_READ);
1299 if (error) {
1300 error = EPERM;
1301 goto trfpout;
1302 }
1303 #endif
1304
1305 /* If we aren't root and target's task access port is set... */
1306 if (!kauth_cred_issuser(kauth_cred_get()) &&
1307 !is_current_proc &&
1308 (task_get_task_access_port(task_read, &tfpport) == 0) &&
1309 (tfpport != IPC_PORT_NULL)) {
1310 if (tfpport == IPC_PORT_DEAD) {
1311 error = EACCES;
1312 goto trfpout;
1313 }
1314
1315
1316 /* Call up to the task access server */
1317 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport,
1318 proc_selfpid(), kauth_getgid(), pid, TASK_FLAVOR_READ);
1319
1320 if (error != MACH_MSG_SUCCESS) {
1321 if (error == MACH_RCV_INTERRUPTED) {
1322 error = EINTR;
1323 } else {
1324 error = EPERM;
1325 }
1326 goto trfpout;
1327 }
1328 }
1329
1330 /* Check if the task has been corpsified */
1331 if (is_corpsetask(task_read)) {
1332 error = EACCES;
1333 goto trfpout;
1334 }
1335
1336 /* could be IP_NULL, consumes a ref */
1337 sright = (void*) convert_task_read_to_port(task_read);
1338 task_read = TASK_READ_NULL;
1339 tret = ipc_port_copyout_send(sright, get_task_ipcspace(current_task()));
1340
1341 trfpout:
1342 task_deallocate(t1);
1343 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1344 if (proc != PROC_NULL) {
1345 proc_rele(proc);
1346 }
1347 if (tfpport != IPC_PORT_NULL) {
1348 ipc_port_release_send(tfpport);
1349 }
1350 if (task_read != TASK_READ_NULL) {
1351 task_deallocate(task_read);
1352 }
1353
1354 *ret = error;
1355 return error;
1356 }
1357
1358 kern_return_t
1359 pid_suspend(struct proc *p __unused, struct pid_suspend_args *args, int *ret)
1360 {
1361 task_t target = NULL;
1362 proc_t targetproc = PROC_NULL;
1363 int pid = args->pid;
1364 int error = 0;
1365 mach_port_t tfpport = MACH_PORT_NULL;
1366
1367 if (pid == 0) {
1368 error = EPERM;
1369 goto out;
1370 }
1371
1372 targetproc = proc_find(pid);
1373 if (targetproc == PROC_NULL) {
1374 error = ESRCH;
1375 goto out;
1376 }
1377
1378 if (!task_for_pid_posix_check(targetproc) &&
1379 !IOTaskHasEntitlement(current_task(), PROCESS_RESUME_SUSPEND_ENTITLEMENT)) {
1380 error = EPERM;
1381 goto out;
1382 }
1383
1384 #if CONFIG_MACF
1385 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_SUSPEND);
1386 if (error) {
1387 error = EPERM;
1388 goto out;
1389 }
1390 #endif
1391
1392 target = targetproc->task;
1393 #if XNU_TARGET_OS_OSX
1394 if (target != TASK_NULL) {
1395 /* If we aren't root and target's task access port is set... */
1396 if (!kauth_cred_issuser(kauth_cred_get()) &&
1397 targetproc != current_proc() &&
1398 (task_get_task_access_port(target, &tfpport) == 0) &&
1399 (tfpport != IPC_PORT_NULL)) {
1400 if (tfpport == IPC_PORT_DEAD) {
1401 error = EACCES;
1402 goto out;
1403 }
1404
1405 /* Call up to the task access server */
1406 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport,
1407 proc_selfpid(), kauth_getgid(), pid, TASK_FLAVOR_CONTROL);
1408
1409 if (error != MACH_MSG_SUCCESS) {
1410 if (error == MACH_RCV_INTERRUPTED) {
1411 error = EINTR;
1412 } else {
1413 error = EPERM;
1414 }
1415 goto out;
1416 }
1417 }
1418 }
1419 #endif /* XNU_TARGET_OS_OSX */
1420
1421 task_reference(target);
1422 error = task_pidsuspend(target);
1423 if (error) {
1424 if (error == KERN_INVALID_ARGUMENT) {
1425 error = EINVAL;
1426 } else {
1427 error = EPERM;
1428 }
1429 }
1430 #if CONFIG_MEMORYSTATUS
1431 else {
1432 memorystatus_on_suspend(targetproc);
1433 }
1434 #endif
1435
1436 task_deallocate(target);
1437
1438 out:
1439 if (tfpport != IPC_PORT_NULL) {
1440 ipc_port_release_send(tfpport);
1441 }
1442
1443 if (targetproc != PROC_NULL) {
1444 proc_rele(targetproc);
1445 }
1446 *ret = error;
1447 return error;
1448 }
1449
1450 kern_return_t
1451 debug_control_port_for_pid(struct debug_control_port_for_pid_args *args)
1452 {
1453 mach_port_name_t target_tport = args->target_tport;
1454 int pid = args->pid;
1455 user_addr_t task_addr = args->t;
1456 proc_t p = PROC_NULL;
1457 task_t t1 = TASK_NULL;
1458 task_t task = TASK_NULL;
1459 mach_port_name_t tret = MACH_PORT_NULL;
1460 ipc_port_t tfpport = MACH_PORT_NULL;
1461 ipc_port_t sright = NULL;
1462 int error = 0;
1463 boolean_t is_current_proc = FALSE;
1464 struct proc_ident pident = {0};
1465
1466 AUDIT_MACH_SYSCALL_ENTER(AUE_DBGPORTFORPID);
1467 AUDIT_ARG(pid, pid);
1468 AUDIT_ARG(mach_port1, target_tport);
1469
1470 /* Always check if pid == 0 */
1471 if (pid == 0) {
1472 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
1473 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
1474 return KERN_FAILURE;
1475 }
1476
1477 t1 = port_name_to_task(target_tport);
1478 if (t1 == TASK_NULL) {
1479 (void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
1480 AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
1481 return KERN_FAILURE;
1482 }
1483
1484 p = proc_find(pid);
1485 if (p == PROC_NULL) {
1486 error = KERN_FAILURE;
1487 goto tfpout;
1488 }
1489 pident = proc_ident(p);
1490 is_current_proc = (p == current_proc());
1491
1492 #if CONFIG_AUDIT
1493 AUDIT_ARG(process, p);
1494 #endif
1495
1496 if (!(task_for_pid_posix_check(p))) {
1497 error = KERN_FAILURE;
1498 goto tfpout;
1499 }
1500
1501 if (p->task == TASK_NULL) {
1502 error = KERN_SUCCESS;
1503 goto tfpout;
1504 }
1505
1506 /*
1507 * Grab a task reference and drop the proc reference before making any upcalls.
1508 */
1509 task = p->task;
1510 task_reference(task);
1511
1512 proc_rele(p);
1513 p = PROC_NULL;
1514
1515 if (!IOTaskHasEntitlement(current_task(), DEBUG_PORT_ENTITLEMENT)) {
1516 #if CONFIG_MACF
1517 error = mac_proc_check_get_task(kauth_cred_get(), &pident, TASK_FLAVOR_CONTROL);
1518 if (error) {
1519 error = KERN_FAILURE;
1520 goto tfpout;
1521 }
1522 #endif
1523
1524 /* If we aren't root and target's task access port is set... */
1525 if (!kauth_cred_issuser(kauth_cred_get()) &&
1526 !is_current_proc &&
1527 (task_get_task_access_port(task, &tfpport) == 0) &&
1528 (tfpport != IPC_PORT_NULL)) {
1529 if (tfpport == IPC_PORT_DEAD) {
1530 error = KERN_PROTECTION_FAILURE;
1531 goto tfpout;
1532 }
1533
1534
1535 /* Call up to the task access server */
1536 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport,
1537 proc_selfpid(), kauth_getgid(), pid, TASK_FLAVOR_CONTROL);
1538
1539 if (error != MACH_MSG_SUCCESS) {
1540 if (error == MACH_RCV_INTERRUPTED) {
1541 error = KERN_ABORTED;
1542 } else {
1543 error = KERN_FAILURE;
1544 }
1545 goto tfpout;
1546 }
1547 }
1548 }
1549
1550 /* Check if the task has been corpsified */
1551 if (is_corpsetask(task)) {
1552 error = KERN_FAILURE;
1553 goto tfpout;
1554 }
1555
1556 error = task_get_debug_control_port(task, &sright);
1557 if (error != KERN_SUCCESS) {
1558 goto tfpout;
1559 }
1560
1561 tret = ipc_port_copyout_send(
1562 sright,
1563 get_task_ipcspace(current_task()));
1564
1565 error = KERN_SUCCESS;
1566
1567 tfpout:
1568 task_deallocate(t1);
1569 AUDIT_ARG(mach_port2, tret);
1570 (void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
1571
1572 if (tfpport != IPC_PORT_NULL) {
1573 ipc_port_release_send(tfpport);
1574 }
1575 if (task != TASK_NULL) {
1576 task_deallocate(task);
1577 }
1578 if (p != PROC_NULL) {
1579 proc_rele(p);
1580 }
1581 AUDIT_MACH_SYSCALL_EXIT(error);
1582 return error;
1583 }
1584
1585 kern_return_t
1586 pid_resume(struct proc *p __unused, struct pid_resume_args *args, int *ret)
1587 {
1588 task_t target = NULL;
1589 proc_t targetproc = PROC_NULL;
1590 int pid = args->pid;
1591 int error = 0;
1592 mach_port_t tfpport = MACH_PORT_NULL;
1593
1594 if (pid == 0) {
1595 error = EPERM;
1596 goto out;
1597 }
1598
1599 targetproc = proc_find(pid);
1600 if (targetproc == PROC_NULL) {
1601 error = ESRCH;
1602 goto out;
1603 }
1604
1605 if (!task_for_pid_posix_check(targetproc) &&
1606 !IOTaskHasEntitlement(current_task(), PROCESS_RESUME_SUSPEND_ENTITLEMENT)) {
1607 error = EPERM;
1608 goto out;
1609 }
1610
1611 #if CONFIG_MACF
1612 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_RESUME);
1613 if (error) {
1614 error = EPERM;
1615 goto out;
1616 }
1617 #endif
1618
1619 target = targetproc->task;
1620 #if XNU_TARGET_OS_OSX
1621 if (target != TASK_NULL) {
1622 /* If we aren't root and target's task access port is set... */
1623 if (!kauth_cred_issuser(kauth_cred_get()) &&
1624 targetproc != current_proc() &&
1625 (task_get_task_access_port(target, &tfpport) == 0) &&
1626 (tfpport != IPC_PORT_NULL)) {
1627 if (tfpport == IPC_PORT_DEAD) {
1628 error = EACCES;
1629 goto out;
1630 }
1631
1632 /* Call up to the task access server */
1633 error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport,
1634 proc_selfpid(), kauth_getgid(), pid, TASK_FLAVOR_CONTROL);
1635
1636 if (error != MACH_MSG_SUCCESS) {
1637 if (error == MACH_RCV_INTERRUPTED) {
1638 error = EINTR;
1639 } else {
1640 error = EPERM;
1641 }
1642 goto out;
1643 }
1644 }
1645 }
1646 #endif /* XNU_TARGET_OS_OSX */
1647
1648 #if !XNU_TARGET_OS_OSX
1649 #if SOCKETS
1650 resume_proc_sockets(targetproc);
1651 #endif /* SOCKETS */
1652 #endif /* !XNU_TARGET_OS_OSX */
1653
1654 task_reference(target);
1655
1656 #if CONFIG_MEMORYSTATUS
1657 memorystatus_on_resume(targetproc);
1658 #endif
1659
1660 error = task_pidresume(target);
1661 if (error) {
1662 if (error == KERN_INVALID_ARGUMENT) {
1663 error = EINVAL;
1664 } else {
1665 if (error == KERN_MEMORY_ERROR) {
1666 psignal(targetproc, SIGKILL);
1667 error = EIO;
1668 } else {
1669 error = EPERM;
1670 }
1671 }
1672 }
1673
1674 task_deallocate(target);
1675
1676 out:
1677 if (tfpport != IPC_PORT_NULL) {
1678 ipc_port_release_send(tfpport);
1679 }
1680
1681 if (targetproc != PROC_NULL) {
1682 proc_rele(targetproc);
1683 }
1684
1685 *ret = error;
1686 return error;
1687 }
1688
1689 #if !XNU_TARGET_OS_OSX
1690 /*
1691 * Freeze the specified process (provided in args->pid), or find and freeze a PID.
1692 * When a process is specified, this call is blocking, otherwise we wake up the
1693 * freezer thread and do not block on a process being frozen.
1694 */
1695 kern_return_t
1696 pid_hibernate(struct proc *p __unused, struct pid_hibernate_args *args, int *ret)
1697 {
1698 int error = 0;
1699 proc_t targetproc = PROC_NULL;
1700 int pid = args->pid;
1701
1702 #ifndef CONFIG_FREEZE
1703 #pragma unused(pid)
1704 #else
1705
1706 /*
1707 * If a pid has been provided, we obtain the process handle and call task_for_pid_posix_check().
1708 */
1709
1710 if (pid >= 0) {
1711 targetproc = proc_find(pid);
1712
1713 if (targetproc == PROC_NULL) {
1714 error = ESRCH;
1715 goto out;
1716 }
1717
1718 if (!task_for_pid_posix_check(targetproc)) {
1719 error = EPERM;
1720 goto out;
1721 }
1722 }
1723
1724 #if CONFIG_MACF
1725 //Note that targetproc may be null
1726 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_HIBERNATE);
1727 if (error) {
1728 error = EPERM;
1729 goto out;
1730 }
1731 #endif
1732
1733 if (pid == -2) {
1734 vm_pageout_anonymous_pages();
1735 } else if (pid == -1) {
1736 memorystatus_on_inactivity(targetproc);
1737 } else {
1738 error = memorystatus_freeze_process_sync(targetproc);
1739 }
1740
1741 out:
1742
1743 #endif /* CONFIG_FREEZE */
1744
1745 if (targetproc != PROC_NULL) {
1746 proc_rele(targetproc);
1747 }
1748 *ret = error;
1749 return error;
1750 }
1751 #endif /* !XNU_TARGET_OS_OSX */
1752
1753 #if SOCKETS
1754 int
1755 networking_memstatus_callout(proc_t p, uint32_t status)
1756 {
1757 struct fileproc *fp;
1758
1759 /*
1760 * proc list lock NOT held
1761 * proc lock NOT held
1762 * a reference on the proc has been held / shall be dropped by the caller.
1763 */
1764 LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED);
1765 LCK_MTX_ASSERT(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED);
1766
1767 proc_fdlock(p);
1768
1769 fdt_foreach(fp, p) {
1770 switch (FILEGLOB_DTYPE(fp->fp_glob)) {
1771 #if NECP
1772 case DTYPE_NETPOLICY:
1773 necp_fd_memstatus(p, status,
1774 (struct necp_fd_data *)fp->fp_glob->fg_data);
1775 break;
1776 #endif /* NECP */
1777 default:
1778 break;
1779 }
1780 }
1781 proc_fdunlock(p);
1782
1783 return 1;
1784 }
1785
1786
1787 static int
1788 networking_defunct_callout(proc_t p, void *arg)
1789 {
1790 struct pid_shutdown_sockets_args *args = arg;
1791 int pid = args->pid;
1792 int level = args->level;
1793 struct fileproc *fp;
1794
1795 proc_fdlock(p);
1796
1797 fdt_foreach(fp, p) {
1798 struct fileglob *fg = fp->fp_glob;
1799
1800 switch (FILEGLOB_DTYPE(fg)) {
1801 case DTYPE_SOCKET: {
1802 struct socket *so = (struct socket *)fg->fg_data;
1803 if (p->p_pid == pid || so->last_pid == pid ||
1804 ((so->so_flags & SOF_DELEGATED) && so->e_pid == pid)) {
1805 /* Call networking stack with socket and level */
1806 (void)socket_defunct(p, so, level);
1807 }
1808 break;
1809 }
1810 #if NECP
1811 case DTYPE_NETPOLICY:
1812 /* first pass: defunct necp and get stats for ntstat */
1813 if (p->p_pid == pid) {
1814 necp_fd_defunct(p,
1815 (struct necp_fd_data *)fg->fg_data);
1816 }
1817 break;
1818 #endif /* NECP */
1819 default:
1820 break;
1821 }
1822 }
1823
1824 proc_fdunlock(p);
1825
1826 return PROC_RETURNED;
1827 }
1828
1829 int
1830 pid_shutdown_sockets(struct proc *p __unused, struct pid_shutdown_sockets_args *args, int *ret)
1831 {
1832 int error = 0;
1833 proc_t targetproc = PROC_NULL;
1834 int pid = args->pid;
1835 int level = args->level;
1836
1837 if (level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_SVC &&
1838 level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL) {
1839 error = EINVAL;
1840 goto out;
1841 }
1842
1843 targetproc = proc_find(pid);
1844 if (targetproc == PROC_NULL) {
1845 error = ESRCH;
1846 goto out;
1847 }
1848
1849 if (!task_for_pid_posix_check(targetproc) &&
1850 !IOTaskHasEntitlement(current_task(), PROCESS_RESUME_SUSPEND_ENTITLEMENT)) {
1851 error = EPERM;
1852 goto out;
1853 }
1854
1855 #if CONFIG_MACF
1856 error = mac_proc_check_suspend_resume(targetproc, MAC_PROC_CHECK_SHUTDOWN_SOCKETS);
1857 if (error) {
1858 error = EPERM;
1859 goto out;
1860 }
1861 #endif
1862
1863 proc_iterate(PROC_ALLPROCLIST | PROC_NOWAITTRANS,
1864 networking_defunct_callout, args, NULL, NULL);
1865
1866 out:
1867 if (targetproc != PROC_NULL) {
1868 proc_rele(targetproc);
1869 }
1870 *ret = error;
1871 return error;
1872 }
1873
1874 #endif /* SOCKETS */
1875
1876 static int
1877 sysctl_settfp_policy(__unused struct sysctl_oid *oidp, void *arg1,
1878 __unused int arg2, struct sysctl_req *req)
1879 {
1880 int error = 0;
1881 int new_value;
1882
1883 error = SYSCTL_OUT(req, arg1, sizeof(int));
1884 if (error || req->newptr == USER_ADDR_NULL) {
1885 return error;
1886 }
1887
1888 if (!kauth_cred_issuser(kauth_cred_get())) {
1889 return EPERM;
1890 }
1891
1892 if ((error = SYSCTL_IN(req, &new_value, sizeof(int)))) {
1893 goto out;
1894 }
1895 if ((new_value == KERN_TFP_POLICY_DENY)
1896 || (new_value == KERN_TFP_POLICY_DEFAULT)) {
1897 tfp_policy = new_value;
1898 } else {
1899 error = EINVAL;
1900 }
1901 out:
1902 return error;
1903 }
1904
1905 #if defined(SECURE_KERNEL)
1906 static int kern_secure_kernel = 1;
1907 #else
1908 static int kern_secure_kernel = 0;
1909 #endif
1910
1911 SYSCTL_INT(_kern, OID_AUTO, secure_kernel, CTLFLAG_RD | CTLFLAG_LOCKED, &kern_secure_kernel, 0, "");
1912
1913 SYSCTL_NODE(_kern, KERN_TFP, tfp, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "tfp");
1914 SYSCTL_PROC(_kern_tfp, KERN_TFP_POLICY, policy, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
1915 &tfp_policy, sizeof(uint32_t), &sysctl_settfp_policy, "I", "policy");
1916
1917 SYSCTL_INT(_vm, OID_AUTO, shared_region_trace_level, CTLFLAG_RW | CTLFLAG_LOCKED,
1918 &shared_region_trace_level, 0, "");
1919 SYSCTL_INT(_vm, OID_AUTO, shared_region_version, CTLFLAG_RD | CTLFLAG_LOCKED,
1920 &shared_region_version, 0, "");
1921 SYSCTL_INT(_vm, OID_AUTO, shared_region_persistence, CTLFLAG_RW | CTLFLAG_LOCKED,
1922 &shared_region_persistence, 0, "");
1923
1924 /*
1925 * shared_region_check_np:
1926 *
1927 * This system call is intended for dyld.
1928 *
1929 * dyld calls this when any process starts to see if the process's shared
1930 * region is already set up and ready to use.
1931 * This call returns the base address of the first mapping in the
1932 * process's shared region's first mapping.
1933 * dyld will then check what's mapped at that address.
1934 *
1935 * If the shared region is empty, dyld will then attempt to map the shared
1936 * cache file in the shared region via the shared_region_map_np() system call.
1937 *
1938 * If something's already mapped in the shared region, dyld will check if it
1939 * matches the shared cache it would like to use for that process.
1940 * If it matches, evrything's ready and the process can proceed and use the
1941 * shared region.
1942 * If it doesn't match, dyld will unmap the shared region and map the shared
1943 * cache into the process's address space via mmap().
1944 *
1945 * ERROR VALUES
1946 * EINVAL no shared region
1947 * ENOMEM shared region is empty
1948 * EFAULT bad address for "start_address"
1949 */
1950 int
1951 shared_region_check_np(
1952 __unused struct proc *p,
1953 struct shared_region_check_np_args *uap,
1954 __unused int *retvalp)
1955 {
1956 vm_shared_region_t shared_region;
1957 mach_vm_offset_t start_address = 0;
1958 int error = 0;
1959 kern_return_t kr;
1960 task_t task = current_task();
1961
1962 SHARED_REGION_TRACE_DEBUG(
1963 ("shared_region: %p [%d(%s)] -> check_np(0x%llx)\n",
1964 (void *)VM_KERNEL_ADDRPERM(current_thread()),
1965 p->p_pid, p->p_comm,
1966 (uint64_t)uap->start_address));
1967
1968 /* retrieve the current tasks's shared region */
1969 shared_region = vm_shared_region_get(task);
1970 if (shared_region != NULL) {
1971 /* retrieve address of its first mapping... */
1972 kr = vm_shared_region_start_address(shared_region, &start_address, task);
1973 if (kr != KERN_SUCCESS) {
1974 error = ENOMEM;
1975 } else {
1976 #if __has_feature(ptrauth_calls)
1977 /*
1978 * Remap any section of the shared library that
1979 * has authenticated pointers into private memory.
1980 */
1981 if (vm_shared_region_auth_remap(shared_region) != KERN_SUCCESS) {
1982 error = ENOMEM;
1983 }
1984 #endif /* __has_feature(ptrauth_calls) */
1985
1986 /* ... and give it to the caller */
1987 if (error == 0) {
1988 error = copyout(&start_address,
1989 (user_addr_t) uap->start_address,
1990 sizeof(start_address));
1991 }
1992 if (error != 0) {
1993 SHARED_REGION_TRACE_ERROR(
1994 ("shared_region: %p [%d(%s)] "
1995 "check_np(0x%llx) "
1996 "copyout(0x%llx) error %d\n",
1997 (void *)VM_KERNEL_ADDRPERM(current_thread()),
1998 p->p_pid, p->p_comm,
1999 (uint64_t)uap->start_address, (uint64_t)start_address,
2000 error));
2001 }
2002 }
2003 vm_shared_region_deallocate(shared_region);
2004 } else {
2005 /* no shared region ! */
2006 error = EINVAL;
2007 }
2008
2009 SHARED_REGION_TRACE_DEBUG(
2010 ("shared_region: %p [%d(%s)] check_np(0x%llx) <- 0x%llx %d\n",
2011 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2012 p->p_pid, p->p_comm,
2013 (uint64_t)uap->start_address, (uint64_t)start_address, error));
2014
2015 return error;
2016 }
2017
2018
2019 static int
2020 shared_region_copyin(
2021 struct proc *p,
2022 user_addr_t user_addr,
2023 unsigned int count,
2024 unsigned int element_size,
2025 void *kernel_data)
2026 {
2027 int error = 0;
2028 vm_size_t size = count * element_size;
2029
2030 error = copyin(user_addr, kernel_data, size);
2031 if (error) {
2032 SHARED_REGION_TRACE_ERROR(
2033 ("shared_region: %p [%d(%s)] map(): "
2034 "copyin(0x%llx, %ld) failed (error=%d)\n",
2035 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2036 p->p_pid, p->p_comm,
2037 (uint64_t)user_addr, (long)size, error));
2038 }
2039 return error;
2040 }
2041
2042 #define _SR_FILE_MAPPINGS_MAX_FILES 2
2043
2044 /* forward declaration */
2045 __attribute__((noinline))
2046 static void shared_region_map_and_slide_cleanup(
2047 struct proc *p,
2048 uint32_t files_count,
2049 struct _sr_file_mappings *sr_file_mappings,
2050 struct vm_shared_region *shared_region,
2051 struct vnode *scdir_vp);
2052
2053 /*
2054 * Setup part of _shared_region_map_and_slide().
2055 * It had to be broken out of _shared_region_map_and_slide() to
2056 * prevent compiler inlining from blowing out the stack.
2057 */
2058 __attribute__((noinline))
2059 static int
2060 shared_region_map_and_slide_setup(
2061 struct proc *p,
2062 uint32_t files_count,
2063 struct shared_file_np *files,
2064 uint32_t mappings_count,
2065 struct shared_file_mapping_slide_np *mappings,
2066 struct _sr_file_mappings **sr_file_mappings,
2067 struct vm_shared_region **shared_region_ptr,
2068 struct vnode **scdir_vp,
2069 struct vnode *rdir_vp)
2070 {
2071 int error = 0;
2072 struct _sr_file_mappings *srfmp;
2073 uint32_t mappings_next;
2074 struct vnode_attr va;
2075 off_t fs;
2076 #if CONFIG_MACF
2077 vm_prot_t maxprot = VM_PROT_ALL;
2078 #endif
2079 uint32_t i;
2080 struct vm_shared_region *shared_region;
2081
2082 SHARED_REGION_TRACE_DEBUG(
2083 ("shared_region: %p [%d(%s)] -> map\n",
2084 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2085 p->p_pid, p->p_comm));
2086
2087 if (files_count > _SR_FILE_MAPPINGS_MAX_FILES) {
2088 error = E2BIG;
2089 goto done;
2090 }
2091 if (files_count == 0) {
2092 error = EINVAL;
2093 goto done;
2094 }
2095 *sr_file_mappings = kheap_alloc(KHEAP_TEMP, files_count * sizeof(struct _sr_file_mappings), Z_WAITOK);
2096 if (*sr_file_mappings == NULL) {
2097 error = ENOMEM;
2098 goto done;
2099 }
2100 bzero(*sr_file_mappings, files_count * sizeof(struct _sr_file_mappings));
2101 mappings_next = 0;
2102 for (i = 0; i < files_count; i++) {
2103 srfmp = &(*sr_file_mappings)[i];
2104 srfmp->fd = files[i].sf_fd;
2105 srfmp->mappings_count = files[i].sf_mappings_count;
2106 srfmp->mappings = &mappings[mappings_next];
2107 mappings_next += srfmp->mappings_count;
2108 if (mappings_next > mappings_count) {
2109 error = EINVAL;
2110 goto done;
2111 }
2112 srfmp->slide = files[i].sf_slide;
2113 }
2114
2115 if (scdir_enforce) {
2116 /* get vnode for scdir_path */
2117 error = vnode_lookup(scdir_path, 0, scdir_vp, vfs_context_current());
2118 if (error) {
2119 SHARED_REGION_TRACE_ERROR(
2120 ("shared_region: %p [%d(%s)]: "
2121 "vnode_lookup(%s) failed (error=%d)\n",
2122 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2123 p->p_pid, p->p_comm,
2124 scdir_path, error));
2125 goto done;
2126 }
2127 }
2128
2129 /* get the process's shared region (setup in vm_map_exec()) */
2130 shared_region = vm_shared_region_trim_and_get(current_task());
2131 *shared_region_ptr = shared_region;
2132 if (shared_region == NULL) {
2133 SHARED_REGION_TRACE_ERROR(
2134 ("shared_region: %p [%d(%s)] map(): "
2135 "no shared region\n",
2136 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2137 p->p_pid, p->p_comm));
2138 error = EINVAL;
2139 goto done;
2140 }
2141
2142 /*
2143 * Check the shared region matches the current root
2144 * directory of this process. Deny the mapping to
2145 * avoid tainting the shared region with something that
2146 * doesn't quite belong into it.
2147 */
2148 struct vnode *sr_vnode = vm_shared_region_root_dir(shared_region);
2149 if (sr_vnode != NULL ? rdir_vp != sr_vnode : rdir_vp != rootvnode) {
2150 SHARED_REGION_TRACE_ERROR(
2151 ("shared_region: map(%p) root_dir mismatch\n",
2152 (void *)VM_KERNEL_ADDRPERM(current_thread())));
2153 error = EPERM;
2154 goto done;
2155 }
2156
2157
2158 for (srfmp = &(*sr_file_mappings)[0];
2159 srfmp < &(*sr_file_mappings)[files_count];
2160 srfmp++) {
2161 if (srfmp->mappings_count == 0) {
2162 /* no mappings here... */
2163 continue;
2164 }
2165
2166 /* get file structure from file descriptor */
2167 error = fp_get_ftype(p, srfmp->fd, DTYPE_VNODE, EINVAL, &srfmp->fp);
2168 if (error) {
2169 SHARED_REGION_TRACE_ERROR(
2170 ("shared_region: %p [%d(%s)] map: "
2171 "fd=%d lookup failed (error=%d)\n",
2172 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2173 p->p_pid, p->p_comm, srfmp->fd, error));
2174 goto done;
2175 }
2176
2177 /* we need at least read permission on the file */
2178 if (!(srfmp->fp->fp_glob->fg_flag & FREAD)) {
2179 SHARED_REGION_TRACE_ERROR(
2180 ("shared_region: %p [%d(%s)] map: "
2181 "fd=%d not readable\n",
2182 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2183 p->p_pid, p->p_comm, srfmp->fd));
2184 error = EPERM;
2185 goto done;
2186 }
2187
2188 /* get vnode from file structure */
2189 error = vnode_getwithref((vnode_t) srfmp->fp->fp_glob->fg_data);
2190 if (error) {
2191 SHARED_REGION_TRACE_ERROR(
2192 ("shared_region: %p [%d(%s)] map: "
2193 "fd=%d getwithref failed (error=%d)\n",
2194 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2195 p->p_pid, p->p_comm, srfmp->fd, error));
2196 goto done;
2197 }
2198 srfmp->vp = (struct vnode *) srfmp->fp->fp_glob->fg_data;
2199
2200 /* make sure the vnode is a regular file */
2201 if (srfmp->vp->v_type != VREG) {
2202 SHARED_REGION_TRACE_ERROR(
2203 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2204 "not a file (type=%d)\n",
2205 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2206 p->p_pid, p->p_comm,
2207 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2208 srfmp->vp->v_name, srfmp->vp->v_type));
2209 error = EINVAL;
2210 goto done;
2211 }
2212
2213 #if CONFIG_MACF
2214 /* pass in 0 for the offset argument because AMFI does not need the offset
2215 * of the shared cache */
2216 error = mac_file_check_mmap(vfs_context_ucred(vfs_context_current()),
2217 srfmp->fp->fp_glob, VM_PROT_ALL, MAP_FILE, 0, &maxprot);
2218 if (error) {
2219 goto done;
2220 }
2221 #endif /* MAC */
2222
2223 #if XNU_TARGET_OS_OSX && defined(__arm64__)
2224 /*
2225 * Check if the shared cache is in the trust cache;
2226 * if so, we can skip the root ownership check.
2227 */
2228 #if DEVELOPMENT || DEBUG
2229 /*
2230 * Skip both root ownership and trust cache check if
2231 * enforcement is disabled.
2232 */
2233 if (!cs_system_enforcement()) {
2234 goto after_root_check;
2235 }
2236 #endif /* DEVELOPMENT || DEBUG */
2237 struct cs_blob *blob = csvnode_get_blob(srfmp->vp, 0);
2238 if (blob == NULL) {
2239 SHARED_REGION_TRACE_ERROR(
2240 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2241 "missing CS blob\n",
2242 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2243 p->p_pid, p->p_comm,
2244 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2245 srfmp->vp->v_name));
2246 goto root_check;
2247 }
2248 const uint8_t *cdhash = csblob_get_cdhash(blob);
2249 if (cdhash == NULL) {
2250 SHARED_REGION_TRACE_ERROR(
2251 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2252 "missing cdhash\n",
2253 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2254 p->p_pid, p->p_comm,
2255 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2256 srfmp->vp->v_name));
2257 goto root_check;
2258 }
2259 uint32_t result = pmap_lookup_in_static_trust_cache(cdhash);
2260 boolean_t in_trust_cache = result & (TC_LOOKUP_FOUND << TC_LOOKUP_RESULT_SHIFT);
2261 if (!in_trust_cache) {
2262 SHARED_REGION_TRACE_ERROR(
2263 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2264 "not in trust cache\n",
2265 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2266 p->p_pid, p->p_comm,
2267 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2268 srfmp->vp->v_name));
2269 goto root_check;
2270 }
2271 goto after_root_check;
2272 root_check:
2273 #endif /* XNU_TARGET_OS_OSX && defined(__arm64__) */
2274
2275 /* The shared cache file must be owned by root */
2276 VATTR_INIT(&va);
2277 VATTR_WANTED(&va, va_uid);
2278 error = vnode_getattr(srfmp->vp, &va, vfs_context_current());
2279 if (error) {
2280 SHARED_REGION_TRACE_ERROR(
2281 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2282 "vnode_getattr(%p) failed (error=%d)\n",
2283 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2284 p->p_pid, p->p_comm,
2285 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2286 srfmp->vp->v_name,
2287 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2288 error));
2289 goto done;
2290 }
2291 if (va.va_uid != 0) {
2292 SHARED_REGION_TRACE_ERROR(
2293 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2294 "owned by uid=%d instead of 0\n",
2295 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2296 p->p_pid, p->p_comm,
2297 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2298 srfmp->vp->v_name, va.va_uid));
2299 error = EPERM;
2300 goto done;
2301 }
2302
2303 #if XNU_TARGET_OS_OSX && defined(__arm64__)
2304 after_root_check:
2305 #endif /* XNU_TARGET_OS_OSX && defined(__arm64__) */
2306
2307 #if CONFIG_CSR
2308 if (csr_check(CSR_ALLOW_UNRESTRICTED_FS) != 0) {
2309 VATTR_INIT(&va);
2310 VATTR_WANTED(&va, va_flags);
2311 error = vnode_getattr(srfmp->vp, &va, vfs_context_current());
2312 if (error) {
2313 SHARED_REGION_TRACE_ERROR(
2314 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2315 "vnode_getattr(%p) failed (error=%d)\n",
2316 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2317 p->p_pid, p->p_comm,
2318 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2319 srfmp->vp->v_name,
2320 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2321 error));
2322 goto done;
2323 }
2324
2325 if (!(va.va_flags & SF_RESTRICTED)) {
2326 /*
2327 * CSR is not configured in CSR_ALLOW_UNRESTRICTED_FS mode, and
2328 * the shared cache file is NOT SIP-protected, so reject the
2329 * mapping request
2330 */
2331 SHARED_REGION_TRACE_ERROR(
2332 ("shared_region: %p [%d(%s)] map(%p:'%s'), "
2333 "vnode is not SIP-protected. \n",
2334 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2335 p->p_pid, p->p_comm,
2336 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2337 srfmp->vp->v_name));
2338 error = EPERM;
2339 goto done;
2340 }
2341 }
2342 #else /* CONFIG_CSR */
2343 /* Devices without SIP/ROSP need to make sure that the shared cache is on the root volume. */
2344
2345 assert(rdir_vp != NULL);
2346 if (srfmp->vp->v_mount != rdir_vp->v_mount) {
2347 SHARED_REGION_TRACE_ERROR(
2348 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2349 "not on process's root volume\n",
2350 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2351 p->p_pid, p->p_comm,
2352 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2353 srfmp->vp->v_name));
2354 error = EPERM;
2355 goto done;
2356 }
2357 #endif /* CONFIG_CSR */
2358
2359 if (scdir_enforce) {
2360 /* ensure parent is scdir_vp */
2361 assert(*scdir_vp != NULL);
2362 if (vnode_parent(srfmp->vp) != *scdir_vp) {
2363 SHARED_REGION_TRACE_ERROR(
2364 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2365 "shared cache file not in %s\n",
2366 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2367 p->p_pid, p->p_comm,
2368 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2369 srfmp->vp->v_name, scdir_path));
2370 error = EPERM;
2371 goto done;
2372 }
2373 }
2374
2375 /* get vnode size */
2376 error = vnode_size(srfmp->vp, &fs, vfs_context_current());
2377 if (error) {
2378 SHARED_REGION_TRACE_ERROR(
2379 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2380 "vnode_size(%p) failed (error=%d)\n",
2381 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2382 p->p_pid, p->p_comm,
2383 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2384 srfmp->vp->v_name,
2385 (void *)VM_KERNEL_ADDRPERM(srfmp->vp), error));
2386 goto done;
2387 }
2388 srfmp->file_size = fs;
2389
2390 /* get the file's memory object handle */
2391 srfmp->file_control = ubc_getobject(srfmp->vp, UBC_HOLDOBJECT);
2392 if (srfmp->file_control == MEMORY_OBJECT_CONTROL_NULL) {
2393 SHARED_REGION_TRACE_ERROR(
2394 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2395 "no memory object\n",
2396 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2397 p->p_pid, p->p_comm,
2398 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2399 srfmp->vp->v_name));
2400 error = EINVAL;
2401 goto done;
2402 }
2403
2404 /* check that the mappings are properly covered by code signatures */
2405 if (!cs_system_enforcement()) {
2406 /* code signing is not enforced: no need to check */
2407 } else {
2408 for (i = 0; i < srfmp->mappings_count; i++) {
2409 if (srfmp->mappings[i].sms_init_prot & VM_PROT_ZF) {
2410 /* zero-filled mapping: not backed by the file */
2411 continue;
2412 }
2413 if (ubc_cs_is_range_codesigned(srfmp->vp,
2414 srfmp->mappings[i].sms_file_offset,
2415 srfmp->mappings[i].sms_size)) {
2416 /* this mapping is fully covered by code signatures */
2417 continue;
2418 }
2419 SHARED_REGION_TRACE_ERROR(
2420 ("shared_region: %p [%d(%s)] map(%p:'%s'): "
2421 "mapping #%d/%d [0x%llx:0x%llx:0x%llx:0x%x:0x%x] "
2422 "is not code-signed\n",
2423 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2424 p->p_pid, p->p_comm,
2425 (void *)VM_KERNEL_ADDRPERM(srfmp->vp),
2426 srfmp->vp->v_name,
2427 i, srfmp->mappings_count,
2428 srfmp->mappings[i].sms_address,
2429 srfmp->mappings[i].sms_size,
2430 srfmp->mappings[i].sms_file_offset,
2431 srfmp->mappings[i].sms_max_prot,
2432 srfmp->mappings[i].sms_init_prot));
2433 error = EINVAL;
2434 goto done;
2435 }
2436 }
2437 }
2438 done:
2439 if (error != 0) {
2440 shared_region_map_and_slide_cleanup(p, files_count, *sr_file_mappings, shared_region, *scdir_vp);
2441 *sr_file_mappings = NULL;
2442 *shared_region_ptr = NULL;
2443 *scdir_vp = NULL;
2444 }
2445 return error;
2446 }
2447
2448 /*
2449 * shared_region_map_np()
2450 *
2451 * This system call is intended for dyld.
2452 *
2453 * dyld uses this to map a shared cache file into a shared region.
2454 * This is usually done only the first time a shared cache is needed.
2455 * Subsequent processes will just use the populated shared region without
2456 * requiring any further setup.
2457 */
2458 static int
2459 _shared_region_map_and_slide(
2460 struct proc *p,
2461 uint32_t files_count,
2462 struct shared_file_np *files,
2463 uint32_t mappings_count,
2464 struct shared_file_mapping_slide_np *mappings)
2465 {
2466 int error = 0;
2467 kern_return_t kr = KERN_SUCCESS;
2468 struct _sr_file_mappings *sr_file_mappings = NULL;
2469 struct vnode *scdir_vp = NULL;
2470 struct vnode *rdir_vp = NULL;
2471 struct vm_shared_region *shared_region = NULL;
2472
2473 /*
2474 * Get a reference to the current proc's root dir.
2475 * Need this to prevent racing with chroot.
2476 */
2477 proc_fdlock(p);
2478 rdir_vp = p->p_fd->fd_rdir;
2479 if (rdir_vp == NULL) {
2480 rdir_vp = rootvnode;
2481 }
2482 assert(rdir_vp != NULL);
2483 vnode_get(rdir_vp);
2484 proc_fdunlock(p);
2485
2486 /*
2487 * Turn files, mappings into sr_file_mappings and other setup.
2488 */
2489 error = shared_region_map_and_slide_setup(p, files_count,
2490 files, mappings_count, mappings,
2491 &sr_file_mappings, &shared_region, &scdir_vp, rdir_vp);
2492 if (error != 0) {
2493 vnode_put(rdir_vp);
2494 return error;
2495 }
2496
2497 /* map the file(s) into that shared region's submap */
2498 kr = vm_shared_region_map_file(shared_region, files_count, sr_file_mappings);
2499 if (kr != KERN_SUCCESS) {
2500 SHARED_REGION_TRACE_ERROR(("shared_region: %p [%d(%s)] map(): "
2501 "vm_shared_region_map_file() failed kr=0x%x\n",
2502 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2503 p->p_pid, p->p_comm, kr));
2504 }
2505
2506 /* convert kern_return_t to errno */
2507 switch (kr) {
2508 case KERN_SUCCESS:
2509 error = 0;
2510 break;
2511 case KERN_INVALID_ADDRESS:
2512 error = EFAULT;
2513 break;
2514 case KERN_PROTECTION_FAILURE:
2515 error = EPERM;
2516 break;
2517 case KERN_NO_SPACE:
2518 error = ENOMEM;
2519 break;
2520 case KERN_FAILURE:
2521 case KERN_INVALID_ARGUMENT:
2522 default:
2523 error = EINVAL;
2524 break;
2525 }
2526
2527 /*
2528 * Mark that this process is now using split libraries.
2529 */
2530 if (error == 0 && (p->p_flag & P_NOSHLIB)) {
2531 OSBitAndAtomic(~((uint32_t)P_NOSHLIB), &p->p_flag);
2532 }
2533
2534 vnode_put(rdir_vp);
2535 shared_region_map_and_slide_cleanup(p, files_count, sr_file_mappings, shared_region, scdir_vp);
2536
2537 SHARED_REGION_TRACE_DEBUG(
2538 ("shared_region: %p [%d(%s)] <- map\n",
2539 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2540 p->p_pid, p->p_comm));
2541
2542 return error;
2543 }
2544
2545 /*
2546 * Clean up part of _shared_region_map_and_slide()
2547 * It had to be broken out of _shared_region_map_and_slide() to
2548 * prevent compiler inlining from blowing out the stack.
2549 */
2550 __attribute__((noinline))
2551 static void
2552 shared_region_map_and_slide_cleanup(
2553 struct proc *p,
2554 uint32_t files_count,
2555 struct _sr_file_mappings *sr_file_mappings,
2556 struct vm_shared_region *shared_region,
2557 struct vnode *scdir_vp)
2558 {
2559 struct _sr_file_mappings *srfmp;
2560 struct vnode_attr va;
2561
2562 if (sr_file_mappings != NULL) {
2563 for (srfmp = &sr_file_mappings[0]; srfmp < &sr_file_mappings[files_count]; srfmp++) {
2564 if (srfmp->vp != NULL) {
2565 vnode_lock_spin(srfmp->vp);
2566 srfmp->vp->v_flag |= VSHARED_DYLD;
2567 vnode_unlock(srfmp->vp);
2568
2569 /* update the vnode's access time */
2570 if (!(vnode_vfsvisflags(srfmp->vp) & MNT_NOATIME)) {
2571 VATTR_INIT(&va);
2572 nanotime(&va.va_access_time);
2573 VATTR_SET_ACTIVE(&va, va_access_time);
2574 vnode_setattr(srfmp->vp, &va, vfs_context_current());
2575 }
2576
2577 #if NAMEDSTREAMS
2578 /*
2579 * If the shared cache is compressed, it may
2580 * have a namedstream vnode instantiated for
2581 * for it. That namedstream vnode will also
2582 * have to be marked with VSHARED_DYLD.
2583 */
2584 if (vnode_hasnamedstreams(srfmp->vp)) {
2585 vnode_t svp;
2586 if (vnode_getnamedstream(srfmp->vp, &svp, XATTR_RESOURCEFORK_NAME,
2587 NS_OPEN, 0, vfs_context_kernel()) == 0) {
2588 vnode_lock_spin(svp);
2589 svp->v_flag |= VSHARED_DYLD;
2590 vnode_unlock(svp);
2591 vnode_put(svp);
2592 }
2593 }
2594 #endif /* NAMEDSTREAMS */
2595 /*
2596 * release the vnode...
2597 * ubc_map() still holds it for us in the non-error case
2598 */
2599 (void) vnode_put(srfmp->vp);
2600 srfmp->vp = NULL;
2601 }
2602 if (srfmp->fp != NULL) {
2603 /* release the file descriptor */
2604 fp_drop(p, srfmp->fd, srfmp->fp, 0);
2605 srfmp->fp = NULL;
2606 }
2607 }
2608 kheap_free(KHEAP_TEMP, sr_file_mappings, files_count * sizeof(*sr_file_mappings));
2609 }
2610
2611 if (scdir_vp != NULL) {
2612 (void)vnode_put(scdir_vp);
2613 scdir_vp = NULL;
2614 }
2615
2616 if (shared_region != NULL) {
2617 vm_shared_region_deallocate(shared_region);
2618 }
2619 }
2620
2621
2622 #define SFM_MAX 1024 /* max mapping structs allowed to pass in */
2623
2624 /*
2625 * This interface is used by dyld to map shared caches which are
2626 * for any architecture which doesn't have run time support of pointer
2627 * authentication. Note dyld could also use the new ...map_and_slide_2_np()
2628 * call for this case, however, it just doesn't do that yet.
2629 */
2630 int
2631 shared_region_map_and_slide_np(
2632 struct proc *p,
2633 struct shared_region_map_and_slide_np_args *uap,
2634 __unused int *retvalp)
2635 {
2636 unsigned int mappings_count = uap->count;
2637 unsigned int m;
2638 uint32_t slide = uap->slide;
2639 struct shared_file_np shared_files[1];
2640 struct shared_file_mapping_np legacy_mapping;
2641 struct shared_file_mapping_slide_np *mappings = NULL;
2642 kern_return_t kr = KERN_SUCCESS;
2643
2644 if ((kr = vm_shared_region_sliding_valid(slide)) != KERN_SUCCESS) {
2645 if (kr == KERN_INVALID_ARGUMENT) {
2646 /*
2647 * This will happen if we request sliding again
2648 * with the same slide value that was used earlier
2649 * for the very first sliding.
2650 */
2651 kr = KERN_SUCCESS;
2652 }
2653 goto done;
2654 }
2655
2656 if (mappings_count == 0) {
2657 SHARED_REGION_TRACE_INFO(
2658 ("shared_region: %p [%d(%s)] map(): "
2659 "no mappings\n",
2660 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2661 p->p_pid, p->p_comm));
2662 kr = 0; /* no mappings: we're done ! */
2663 goto done;
2664 } else if (mappings_count <= SFM_MAX) {
2665 mappings = kheap_alloc(KHEAP_TEMP,
2666 mappings_count * sizeof(mappings[0]), Z_WAITOK);
2667 if (mappings == NULL) {
2668 kr = KERN_RESOURCE_SHORTAGE;
2669 goto done;
2670 }
2671 } else {
2672 SHARED_REGION_TRACE_ERROR(
2673 ("shared_region: %p [%d(%s)] map(): "
2674 "too many mappings (%d) max %d\n",
2675 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2676 p->p_pid, p->p_comm,
2677 mappings_count, SFM_MAX));
2678 kr = KERN_FAILURE;
2679 goto done;
2680 }
2681
2682 /*
2683 * Read in the mappings and translate to new format.
2684 */
2685 for (m = 0; m < mappings_count; ++m) {
2686 user_addr_t from_uaddr = uap->mappings + (m * sizeof(struct shared_file_mapping_np));
2687 kr = shared_region_copyin(p, from_uaddr, 1, sizeof(legacy_mapping), &legacy_mapping);
2688 if (kr != 0) {
2689 goto done;
2690 }
2691 mappings[m].sms_address = legacy_mapping.sfm_address;
2692 mappings[m].sms_size = legacy_mapping.sfm_size;
2693 mappings[m].sms_file_offset = legacy_mapping.sfm_file_offset;
2694 mappings[m].sms_max_prot = legacy_mapping.sfm_max_prot;
2695 mappings[m].sms_init_prot = legacy_mapping.sfm_init_prot;
2696 mappings[m].sms_slide_size = uap->slide_size;
2697 mappings[m].sms_slide_start = uap->slide_start;
2698 }
2699
2700 bzero(shared_files, sizeof(shared_files));
2701 shared_files[0].sf_fd = uap->fd;
2702 shared_files[0].sf_mappings_count = mappings_count;
2703 shared_files[0].sf_slide = slide;
2704
2705 kr = _shared_region_map_and_slide(p,
2706 1, /* # of files to map */
2707 &shared_files[0], /* files to map */
2708 mappings_count,
2709 mappings);
2710
2711 done:
2712 if (mappings != NULL) {
2713 kheap_free(KHEAP_TEMP, mappings, mappings_count * sizeof(mappings[0]));
2714 mappings = NULL;
2715 }
2716 return kr;
2717 }
2718
2719 /*
2720 * This interface for setting up shared region mappings is what dyld
2721 * uses for shared caches that have __AUTH sections. All other shared
2722 * caches use the non _2 version.
2723 *
2724 * The slide used for shared regions setup using this interface is done differently
2725 * from the old interface. The slide value passed in the shared_files_np represents
2726 * a max value. The kernel will choose a random value based on that, then use it
2727 * for all shared regions.
2728 */
2729 #if defined (__x86_64__)
2730 #define SLIDE_AMOUNT_MASK ~FOURK_PAGE_MASK
2731 #else
2732 #define SLIDE_AMOUNT_MASK ~SIXTEENK_PAGE_MASK
2733 #endif
2734
2735 int
2736 shared_region_map_and_slide_2_np(
2737 struct proc *p,
2738 struct shared_region_map_and_slide_2_np_args *uap,
2739 __unused int *retvalp)
2740 {
2741 unsigned int files_count;
2742 struct shared_file_np *shared_files = NULL;
2743 unsigned int mappings_count;
2744 struct shared_file_mapping_slide_np *mappings = NULL;
2745 kern_return_t kr = KERN_SUCCESS;
2746 boolean_t should_slide_mappings = TRUE;
2747
2748 files_count = uap->files_count;
2749 mappings_count = uap->mappings_count;
2750
2751
2752 if (files_count == 0) {
2753 SHARED_REGION_TRACE_INFO(
2754 ("shared_region: %p [%d(%s)] map(): "
2755 "no files\n",
2756 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2757 p->p_pid, p->p_comm));
2758 kr = 0; /* no files to map: we're done ! */
2759 goto done;
2760 } else if (files_count <= _SR_FILE_MAPPINGS_MAX_FILES) {
2761 shared_files = kheap_alloc(KHEAP_TEMP,
2762 files_count * sizeof(shared_files[0]), Z_WAITOK);
2763 if (shared_files == NULL) {
2764 kr = KERN_RESOURCE_SHORTAGE;
2765 goto done;
2766 }
2767 } else {
2768 SHARED_REGION_TRACE_ERROR(
2769 ("shared_region: %p [%d(%s)] map(): "
2770 "too many files (%d) max %d\n",
2771 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2772 p->p_pid, p->p_comm,
2773 files_count, _SR_FILE_MAPPINGS_MAX_FILES));
2774 kr = KERN_FAILURE;
2775 goto done;
2776 }
2777
2778 if (mappings_count == 0) {
2779 SHARED_REGION_TRACE_INFO(
2780 ("shared_region: %p [%d(%s)] map(): "
2781 "no mappings\n",
2782 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2783 p->p_pid, p->p_comm));
2784 kr = 0; /* no mappings: we're done ! */
2785 goto done;
2786 } else if (mappings_count <= SFM_MAX) {
2787 mappings = kheap_alloc(KHEAP_TEMP,
2788 mappings_count * sizeof(mappings[0]), Z_WAITOK);
2789 if (mappings == NULL) {
2790 kr = KERN_RESOURCE_SHORTAGE;
2791 goto done;
2792 }
2793 } else {
2794 SHARED_REGION_TRACE_ERROR(
2795 ("shared_region: %p [%d(%s)] map(): "
2796 "too many mappings (%d) max %d\n",
2797 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2798 p->p_pid, p->p_comm,
2799 mappings_count, SFM_MAX));
2800 kr = KERN_FAILURE;
2801 goto done;
2802 }
2803
2804 kr = shared_region_copyin(p, uap->files, files_count, sizeof(shared_files[0]), shared_files);
2805 if (kr != KERN_SUCCESS) {
2806 goto done;
2807 }
2808
2809 kr = shared_region_copyin(p, uap->mappings, mappings_count, sizeof(mappings[0]), mappings);
2810 if (kr != KERN_SUCCESS) {
2811 goto done;
2812 }
2813
2814 if (should_slide_mappings) {
2815 uint32_t max_slide = shared_files[0].sf_slide;
2816 uint32_t random_val;
2817 uint32_t slide_amount;
2818
2819 if (max_slide != 0) {
2820 read_random(&random_val, sizeof random_val);
2821 slide_amount = ((random_val % max_slide) & SLIDE_AMOUNT_MASK);
2822 } else {
2823 slide_amount = 0;
2824 }
2825
2826 /*
2827 * Fix up the mappings to reflect the desired slide.
2828 */
2829 unsigned int f;
2830 unsigned int m = 0;
2831 unsigned int i;
2832 for (f = 0; f < files_count; ++f) {
2833 shared_files[f].sf_slide = slide_amount;
2834 for (i = 0; i < shared_files[f].sf_mappings_count; ++i, ++m) {
2835 if (m >= mappings_count) {
2836 SHARED_REGION_TRACE_ERROR(
2837 ("shared_region: %p [%d(%s)] map(): "
2838 "mapping count argument was too small\n",
2839 (void *)VM_KERNEL_ADDRPERM(current_thread()),
2840 p->p_pid, p->p_comm));
2841 kr = KERN_FAILURE;
2842 goto done;
2843 }
2844 mappings[m].sms_address += slide_amount;
2845 if (mappings[m].sms_slide_size != 0) {
2846 mappings[m].sms_slide_start += slide_amount;
2847 }
2848 }
2849 }
2850 }
2851 kr = _shared_region_map_and_slide(p, files_count, shared_files, mappings_count, mappings);
2852 done:
2853 if (shared_files != NULL) {
2854 kheap_free(KHEAP_TEMP, shared_files, files_count * sizeof(shared_files[0]));
2855 shared_files = NULL;
2856 }
2857 if (mappings != NULL) {
2858 kheap_free(KHEAP_TEMP, mappings, mappings_count * sizeof(mappings[0]));
2859 mappings = NULL;
2860 }
2861 return kr;
2862 }
2863
2864 /* sysctl overflow room */
2865
2866 SYSCTL_INT(_vm, OID_AUTO, pagesize, CTLFLAG_RD | CTLFLAG_LOCKED,
2867 (int *) &page_size, 0, "vm page size");
2868
2869 /* vm_page_free_target is provided as a makeshift solution for applications that want to
2870 * allocate buffer space, possibly purgeable memory, but not cause inactive pages to be
2871 * reclaimed. It allows the app to calculate how much memory is free outside the free target. */
2872 extern unsigned int vm_page_free_target;
2873 SYSCTL_INT(_vm, OID_AUTO, vm_page_free_target, CTLFLAG_RD | CTLFLAG_LOCKED,
2874 &vm_page_free_target, 0, "Pageout daemon free target");
2875
2876 SYSCTL_INT(_vm, OID_AUTO, memory_pressure, CTLFLAG_RD | CTLFLAG_LOCKED,
2877 &vm_pageout_state.vm_memory_pressure, 0, "Memory pressure indicator");
2878
2879 static int
2880 vm_ctl_page_free_wanted SYSCTL_HANDLER_ARGS
2881 {
2882 #pragma unused(oidp, arg1, arg2)
2883 unsigned int page_free_wanted;
2884
2885 page_free_wanted = mach_vm_ctl_page_free_wanted();
2886 return SYSCTL_OUT(req, &page_free_wanted, sizeof(page_free_wanted));
2887 }
2888 SYSCTL_PROC(_vm, OID_AUTO, page_free_wanted,
2889 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
2890 0, 0, vm_ctl_page_free_wanted, "I", "");
2891
2892 extern unsigned int vm_page_purgeable_count;
2893 SYSCTL_INT(_vm, OID_AUTO, page_purgeable_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2894 &vm_page_purgeable_count, 0, "Purgeable page count");
2895
2896 extern unsigned int vm_page_purgeable_wired_count;
2897 SYSCTL_INT(_vm, OID_AUTO, page_purgeable_wired_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2898 &vm_page_purgeable_wired_count, 0, "Wired purgeable page count");
2899
2900 extern unsigned int vm_page_kern_lpage_count;
2901 SYSCTL_INT(_vm, OID_AUTO, kern_lpage_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2902 &vm_page_kern_lpage_count, 0, "kernel used large pages");
2903
2904 #if DEVELOPMENT || DEBUG
2905 #if __ARM_MIXED_PAGE_SIZE__
2906 static int vm_mixed_pagesize_supported = 1;
2907 #else
2908 static int vm_mixed_pagesize_supported = 0;
2909 #endif /*__ARM_MIXED_PAGE_SIZE__ */
2910 SYSCTL_INT(_debug, OID_AUTO, vm_mixed_pagesize_supported, CTLFLAG_ANYBODY | CTLFLAG_RD | CTLFLAG_LOCKED,
2911 &vm_mixed_pagesize_supported, 0, "kernel support for mixed pagesize");
2912
2913 SCALABLE_COUNTER_DECLARE(vm_page_grab_count);
2914 SYSCTL_SCALABLE_COUNTER(_vm, pages_grabbed, vm_page_grab_count, "Total pages grabbed");
2915 SYSCTL_ULONG(_vm, OID_AUTO, pages_freed, CTLFLAG_RD | CTLFLAG_LOCKED,
2916 &vm_pageout_vminfo.vm_page_pages_freed, "Total pages freed");
2917
2918 SYSCTL_INT(_vm, OID_AUTO, pageout_purged_objects, CTLFLAG_RD | CTLFLAG_LOCKED,
2919 &vm_pageout_debug.vm_pageout_purged_objects, 0, "System purged object count");
2920 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_busy, CTLFLAG_RD | CTLFLAG_LOCKED,
2921 &vm_pageout_debug.vm_pageout_cleaned_busy, 0, "Cleaned pages busy (deactivated)");
2922 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_nolock, CTLFLAG_RD | CTLFLAG_LOCKED,
2923 &vm_pageout_debug.vm_pageout_cleaned_nolock, 0, "Cleaned pages no-lock (deactivated)");
2924
2925 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_volatile_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2926 &vm_pageout_debug.vm_pageout_cleaned_volatile_reactivated, 0, "Cleaned pages volatile reactivated");
2927 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_fault_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2928 &vm_pageout_debug.vm_pageout_cleaned_fault_reactivated, 0, "Cleaned pages fault reactivated");
2929 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2930 &vm_pageout_debug.vm_pageout_cleaned_reactivated, 0, "Cleaned pages reactivated"); /* sum of all reactivated AND busy and nolock (even though those actually get reDEactivated */
2931 SYSCTL_ULONG(_vm, OID_AUTO, pageout_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED,
2932 &vm_pageout_vminfo.vm_pageout_freed_cleaned, "Cleaned pages freed");
2933 SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_reference_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED,
2934 &vm_pageout_debug.vm_pageout_cleaned_reference_reactivated, 0, "Cleaned pages reference reactivated");
2935 SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED,
2936 &vm_pageout_debug.vm_pageout_enqueued_cleaned, 0, ""); /* sum of next two */
2937 #endif /* DEVELOPMENT || DEBUG */
2938
2939 extern int madvise_free_debug;
2940 SYSCTL_INT(_vm, OID_AUTO, madvise_free_debug, CTLFLAG_RW | CTLFLAG_LOCKED,
2941 &madvise_free_debug, 0, "zero-fill on madvise(MADV_FREE*)");
2942
2943 SYSCTL_INT(_vm, OID_AUTO, page_reusable_count, CTLFLAG_RD | CTLFLAG_LOCKED,
2944 &vm_page_stats_reusable.reusable_count, 0, "Reusable page count");
2945 SYSCTL_QUAD(_vm, OID_AUTO, reusable_success, CTLFLAG_RD | CTLFLAG_LOCKED,
2946 &vm_page_stats_reusable.reusable_pages_success, "");
2947 SYSCTL_QUAD(_vm, OID_AUTO, reusable_failure, CTLFLAG_RD | CTLFLAG_LOCKED,
2948 &vm_page_stats_reusable.reusable_pages_failure, "");
2949 SYSCTL_QUAD(_vm, OID_AUTO, reusable_pages_shared, CTLFLAG_RD | CTLFLAG_LOCKED,
2950 &vm_page_stats_reusable.reusable_pages_shared, "");
2951 SYSCTL_QUAD(_vm, OID_AUTO, all_reusable_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2952 &vm_page_stats_reusable.all_reusable_calls, "");
2953 SYSCTL_QUAD(_vm, OID_AUTO, partial_reusable_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2954 &vm_page_stats_reusable.partial_reusable_calls, "");
2955 SYSCTL_QUAD(_vm, OID_AUTO, reuse_success, CTLFLAG_RD | CTLFLAG_LOCKED,
2956 &vm_page_stats_reusable.reuse_pages_success, "");
2957 SYSCTL_QUAD(_vm, OID_AUTO, reuse_failure, CTLFLAG_RD | CTLFLAG_LOCKED,
2958 &vm_page_stats_reusable.reuse_pages_failure, "");
2959 SYSCTL_QUAD(_vm, OID_AUTO, all_reuse_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2960 &vm_page_stats_reusable.all_reuse_calls, "");
2961 SYSCTL_QUAD(_vm, OID_AUTO, partial_reuse_calls, CTLFLAG_RD | CTLFLAG_LOCKED,
2962 &vm_page_stats_reusable.partial_reuse_calls, "");
2963 SYSCTL_QUAD(_vm, OID_AUTO, can_reuse_success, CTLFLAG_RD | CTLFLAG_LOCKED,
2964 &vm_page_stats_reusable.can_reuse_success, "");
2965 SYSCTL_QUAD(_vm, OID_AUTO, can_reuse_failure, CTLFLAG_RD | CTLFLAG_LOCKED,
2966 &vm_page_stats_reusable.can_reuse_failure, "");
2967 SYSCTL_QUAD(_vm, OID_AUTO, reusable_reclaimed, CTLFLAG_RD | CTLFLAG_LOCKED,
2968 &vm_page_stats_reusable.reusable_reclaimed, "");
2969 SYSCTL_QUAD(_vm, OID_AUTO, reusable_nonwritable, CTLFLAG_RD | CTLFLAG_LOCKED,
2970 &vm_page_stats_reusable.reusable_nonwritable, "");
2971 SYSCTL_QUAD(_vm, OID_AUTO, reusable_shared, CTLFLAG_RD | CTLFLAG_LOCKED,
2972 &vm_page_stats_reusable.reusable_shared, "");
2973 SYSCTL_QUAD(_vm, OID_AUTO, free_shared, CTLFLAG_RD | CTLFLAG_LOCKED,
2974 &vm_page_stats_reusable.free_shared, "");
2975
2976
2977 extern unsigned int vm_page_free_count, vm_page_speculative_count;
2978 SYSCTL_UINT(_vm, OID_AUTO, page_free_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_free_count, 0, "");
2979 SYSCTL_UINT(_vm, OID_AUTO, page_speculative_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_speculative_count, 0, "");
2980
2981 extern unsigned int vm_page_cleaned_count;
2982 SYSCTL_UINT(_vm, OID_AUTO, page_cleaned_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_cleaned_count, 0, "Cleaned queue size");
2983
2984 extern unsigned int vm_page_pageable_internal_count, vm_page_pageable_external_count;
2985 SYSCTL_UINT(_vm, OID_AUTO, page_pageable_internal_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pageable_internal_count, 0, "");
2986 SYSCTL_UINT(_vm, OID_AUTO, page_pageable_external_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pageable_external_count, 0, "");
2987
2988 /* pageout counts */
2989 SYSCTL_UINT(_vm, OID_AUTO, pageout_inactive_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_state.vm_pageout_inactive_clean, 0, "");
2990 SYSCTL_UINT(_vm, OID_AUTO, pageout_inactive_used, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_state.vm_pageout_inactive_used, 0, "");
2991
2992 SYSCTL_ULONG(_vm, OID_AUTO, pageout_inactive_dirty_internal, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_inactive_dirty_internal, "");
2993 SYSCTL_ULONG(_vm, OID_AUTO, pageout_inactive_dirty_external, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_inactive_dirty_external, "");
2994 SYSCTL_ULONG(_vm, OID_AUTO, pageout_speculative_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_speculative, "");
2995 SYSCTL_ULONG(_vm, OID_AUTO, pageout_freed_external, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_external, "");
2996 SYSCTL_ULONG(_vm, OID_AUTO, pageout_freed_speculative, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_speculative, "");
2997 SYSCTL_ULONG(_vm, OID_AUTO, pageout_freed_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_vminfo.vm_pageout_freed_cleaned, "");
2998
2999
3000 /* counts of pages prefaulted when entering a memory object */
3001 extern int64_t vm_prefault_nb_pages, vm_prefault_nb_bailout;
3002 SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_pages, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_pages, "");
3003 SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_bailout, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_bailout, "");
3004
3005 #if defined (__x86_64__)
3006 extern unsigned int vm_clump_promote_threshold;
3007 SYSCTL_UINT(_vm, OID_AUTO, vm_clump_promote_threshold, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_clump_promote_threshold, 0, "clump size threshold for promotes");
3008 #if DEVELOPMENT || DEBUG
3009 extern unsigned long vm_clump_stats[];
3010 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats1, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[1], "free page allocations from clump of 1 page");
3011 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats2, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[2], "free page allocations from clump of 2 pages");
3012 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats3, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[3], "free page allocations from clump of 3 pages");
3013 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats4, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[4], "free page allocations from clump of 4 pages");
3014 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats5, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[5], "free page allocations from clump of 5 pages");
3015 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats6, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[6], "free page allocations from clump of 6 pages");
3016 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats7, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[7], "free page allocations from clump of 7 pages");
3017 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats8, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[8], "free page allocations from clump of 8 pages");
3018 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats9, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[9], "free page allocations from clump of 9 pages");
3019 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats10, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[10], "free page allocations from clump of 10 pages");
3020 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats11, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[11], "free page allocations from clump of 11 pages");
3021 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats12, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[12], "free page allocations from clump of 12 pages");
3022 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats13, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[13], "free page allocations from clump of 13 pages");
3023 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats14, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[14], "free page allocations from clump of 14 pages");
3024 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats15, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[15], "free page allocations from clump of 15 pages");
3025 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_stats16, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_stats[16], "free page allocations from clump of 16 pages");
3026 extern unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes;
3027 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_alloc, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_allocs, "free page allocations");
3028 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_inserts, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_inserts, "free page insertions");
3029 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_inrange, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_inrange, "free page insertions that are part of vm_pages");
3030 SYSCTL_LONG(_vm, OID_AUTO, vm_clump_promotes, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_clump_promotes, "pages promoted to head");
3031 #endif /* if DEVELOPMENT || DEBUG */
3032 #endif /* #if defined (__x86_64__) */
3033
3034 #if CONFIG_SECLUDED_MEMORY
3035
3036 SYSCTL_UINT(_vm, OID_AUTO, num_tasks_can_use_secluded_mem, CTLFLAG_RD | CTLFLAG_LOCKED, &num_tasks_can_use_secluded_mem, 0, "");
3037 extern unsigned int vm_page_secluded_target;
3038 extern unsigned int vm_page_secluded_count;
3039 extern unsigned int vm_page_secluded_count_free;
3040 extern unsigned int vm_page_secluded_count_inuse;
3041 extern unsigned int vm_page_secluded_count_over_target;
3042 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_target, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_target, 0, "");
3043 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count, 0, "");
3044 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count_free, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count_free, 0, "");
3045 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count_inuse, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count_inuse, 0, "");
3046 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_count_over_target, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded_count_over_target, 0, "");
3047
3048 extern struct vm_page_secluded_data vm_page_secluded;
3049 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_eligible, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.eligible_for_secluded, 0, "");
3050 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_success_free, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_success_free, 0, "");
3051 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_success_other, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_success_other, 0, "");
3052 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_locked, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_failure_locked, 0, "");
3053 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_state, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_failure_state, 0, "");
3054 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_failure_dirty, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_failure_dirty, 0, "");
3055 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_for_iokit, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_for_iokit, 0, "");
3056 SYSCTL_UINT(_vm, OID_AUTO, page_secluded_grab_for_iokit_success, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_secluded.grab_for_iokit_success, 0, "");
3057
3058 #endif /* CONFIG_SECLUDED_MEMORY */
3059
3060 #include <kern/thread.h>
3061 #include <sys/user.h>
3062
3063 void vm_pageout_io_throttle(void);
3064
3065 void
3066 vm_pageout_io_throttle(void)
3067 {
3068 struct uthread *uthread = get_bsdthread_info(current_thread());
3069
3070 /*
3071 * thread is marked as a low priority I/O type
3072 * and the I/O we issued while in this cleaning operation
3073 * collided with normal I/O operations... we'll
3074 * delay in order to mitigate the impact of this
3075 * task on the normal operation of the system
3076 */
3077
3078 if (uthread->uu_lowpri_window) {
3079 throttle_lowpri_io(1);
3080 }
3081 }
3082
3083 int
3084 vm_pressure_monitor(
3085 __unused struct proc *p,
3086 struct vm_pressure_monitor_args *uap,
3087 int *retval)
3088 {
3089 kern_return_t kr;
3090 uint32_t pages_reclaimed;
3091 uint32_t pages_wanted;
3092
3093 kr = mach_vm_pressure_monitor(
3094 (boolean_t) uap->wait_for_pressure,
3095 uap->nsecs_monitored,
3096 (uap->pages_reclaimed) ? &pages_reclaimed : NULL,
3097 &pages_wanted);
3098
3099 switch (kr) {
3100 case KERN_SUCCESS:
3101 break;
3102 case KERN_ABORTED:
3103 return EINTR;
3104 default:
3105 return EINVAL;
3106 }
3107
3108 if (uap->pages_reclaimed) {
3109 if (copyout((void *)&pages_reclaimed,
3110 uap->pages_reclaimed,
3111 sizeof(pages_reclaimed)) != 0) {
3112 return EFAULT;
3113 }
3114 }
3115
3116 *retval = (int) pages_wanted;
3117 return 0;
3118 }
3119
3120 int
3121 kas_info(struct proc *p,
3122 struct kas_info_args *uap,
3123 int *retval __unused)
3124 {
3125 #ifndef CONFIG_KAS_INFO
3126 (void)p;
3127 (void)uap;
3128 return ENOTSUP;
3129 #else /* CONFIG_KAS_INFO */
3130 int selector = uap->selector;
3131 user_addr_t valuep = uap->value;
3132 user_addr_t sizep = uap->size;
3133 user_size_t size, rsize;
3134 int error;
3135
3136 if (!kauth_cred_issuser(kauth_cred_get())) {
3137 return EPERM;
3138 }
3139
3140 #if CONFIG_MACF
3141 error = mac_system_check_kas_info(kauth_cred_get(), selector);
3142 if (error) {
3143 return error;
3144 }
3145 #endif
3146
3147 if (IS_64BIT_PROCESS(p)) {
3148 user64_size_t size64;
3149 error = copyin(sizep, &size64, sizeof(size64));
3150 size = (user_size_t)size64;
3151 } else {
3152 user32_size_t size32;
3153 error = copyin(sizep, &size32, sizeof(size32));
3154 size = (user_size_t)size32;
3155 }
3156 if (error) {
3157 return error;
3158 }
3159
3160 switch (selector) {
3161 case KAS_INFO_KERNEL_TEXT_SLIDE_SELECTOR:
3162 {
3163 uint64_t slide = vm_kernel_slide;
3164
3165 if (sizeof(slide) != size) {
3166 return EINVAL;
3167 }
3168
3169 error = copyout(&slide, valuep, sizeof(slide));
3170 if (error) {
3171 return error;
3172 }
3173 rsize = size;
3174 }
3175 break;
3176 case KAS_INFO_KERNEL_SEGMENT_VMADDR_SELECTOR:
3177 {
3178 uint32_t i;
3179 kernel_mach_header_t *mh = &_mh_execute_header;
3180 struct load_command *cmd;
3181 cmd = (struct load_command*) &mh[1];
3182 uint64_t *bases;
3183 rsize = mh->ncmds * sizeof(uint64_t);
3184
3185 /*
3186 * Return the size if no data was passed
3187 */
3188 if (valuep == 0) {
3189 break;
3190 }
3191
3192 if (rsize > size) {
3193 return EINVAL;
3194 }
3195
3196 bases = kheap_alloc(KHEAP_TEMP, rsize, Z_WAITOK | Z_ZERO);
3197
3198 for (i = 0; i < mh->ncmds; i++) {
3199 if (cmd->cmd == LC_SEGMENT_KERNEL) {
3200 __IGNORE_WCASTALIGN(kernel_segment_command_t * sg = (kernel_segment_command_t *) cmd);
3201 bases[i] = (uint64_t)sg->vmaddr;
3202 }
3203 cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
3204 }
3205
3206 error = copyout(bases, valuep, rsize);
3207
3208 kheap_free(KHEAP_TEMP, bases, rsize);
3209
3210 if (error) {
3211 return error;
3212 }
3213 }
3214 break;
3215 default:
3216 return EINVAL;
3217 }
3218
3219 if (IS_64BIT_PROCESS(p)) {
3220 user64_size_t size64 = (user64_size_t)rsize;
3221 error = copyout(&size64, sizep, sizeof(size64));
3222 } else {
3223 user32_size_t size32 = (user32_size_t)rsize;
3224 error = copyout(&size32, sizep, sizeof(size32));
3225 }
3226
3227 return error;
3228 #endif /* CONFIG_KAS_INFO */
3229 }
3230
3231 #if __has_feature(ptrauth_calls)
3232 /*
3233 * Generate a random pointer signing key that isn't 0.
3234 */
3235 uint64_t
3236 generate_jop_key(void)
3237 {
3238 uint64_t key;
3239
3240 do {
3241 read_random(&key, sizeof key);
3242 } while (key == 0);
3243 return key;
3244 }
3245 #endif /* __has_feature(ptrauth_calls) */
3246
3247
3248 #pragma clang diagnostic push
3249 #pragma clang diagnostic ignored "-Wcast-qual"
3250 #pragma clang diagnostic ignored "-Wunused-function"
3251
3252 static void
3253 asserts()
3254 {
3255 static_assert(sizeof(vm_min_kernel_address) == sizeof(unsigned long));
3256 static_assert(sizeof(vm_max_kernel_address) == sizeof(unsigned long));
3257 }
3258
3259 SYSCTL_ULONG(_vm, OID_AUTO, vm_min_kernel_address, CTLFLAG_RD, (unsigned long *) &vm_min_kernel_address, "");
3260 SYSCTL_ULONG(_vm, OID_AUTO, vm_max_kernel_address, CTLFLAG_RD, (unsigned long *) &vm_max_kernel_address, "");
3261 #pragma clang diagnostic pop
3262
3263 extern uint32_t vm_page_pages;
3264 SYSCTL_UINT(_vm, OID_AUTO, pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_pages, 0, "");
3265
3266 extern uint32_t vm_page_busy_absent_skipped;
3267 SYSCTL_UINT(_vm, OID_AUTO, page_busy_absent_skipped, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_busy_absent_skipped, 0, "");
3268
3269 extern uint32_t vm_page_upl_tainted;
3270 SYSCTL_UINT(_vm, OID_AUTO, upl_pages_tainted, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_upl_tainted, 0, "");
3271
3272 extern uint32_t vm_page_iopl_tainted;
3273 SYSCTL_UINT(_vm, OID_AUTO, iopl_pages_tainted, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_iopl_tainted, 0, "");
3274
3275 #if (__arm__ || __arm64__) && (DEVELOPMENT || DEBUG)
3276 extern int vm_footprint_suspend_allowed;
3277 SYSCTL_INT(_vm, OID_AUTO, footprint_suspend_allowed, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_footprint_suspend_allowed, 0, "");
3278
3279 extern void pmap_footprint_suspend(vm_map_t map, boolean_t suspend);
3280 static int
3281 sysctl_vm_footprint_suspend SYSCTL_HANDLER_ARGS
3282 {
3283 #pragma unused(oidp, arg1, arg2)
3284 int error = 0;
3285 int new_value;
3286
3287 if (req->newptr == USER_ADDR_NULL) {
3288 return 0;
3289 }
3290 error = SYSCTL_IN(req, &new_value, sizeof(int));
3291 if (error) {
3292 return error;
3293 }
3294 if (!vm_footprint_suspend_allowed) {
3295 if (new_value != 0) {
3296 /* suspends are not allowed... */
3297 return 0;
3298 }
3299 /* ... but let resumes proceed */
3300 }
3301 DTRACE_VM2(footprint_suspend,
3302 vm_map_t, current_map(),
3303 int, new_value);
3304
3305 pmap_footprint_suspend(current_map(), new_value);
3306
3307 return 0;
3308 }
3309 SYSCTL_PROC(_vm, OID_AUTO, footprint_suspend,
3310 CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_ANYBODY | CTLFLAG_LOCKED | CTLFLAG_MASKED,
3311 0, 0, &sysctl_vm_footprint_suspend, "I", "");
3312 #endif /* (__arm__ || __arm64__) && (DEVELOPMENT || DEBUG) */
3313
3314 extern uint64_t vm_map_corpse_footprint_count;
3315 extern uint64_t vm_map_corpse_footprint_size_avg;
3316 extern uint64_t vm_map_corpse_footprint_size_max;
3317 extern uint64_t vm_map_corpse_footprint_full;
3318 extern uint64_t vm_map_corpse_footprint_no_buf;
3319 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_count,
3320 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_count, "");
3321 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_size_avg,
3322 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_size_avg, "");
3323 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_size_max,
3324 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_size_max, "");
3325 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_full,
3326 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_full, "");
3327 SYSCTL_QUAD(_vm, OID_AUTO, corpse_footprint_no_buf,
3328 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_corpse_footprint_no_buf, "");
3329
3330
3331 extern uint64_t shared_region_pager_copied;
3332 extern uint64_t shared_region_pager_slid;
3333 extern uint64_t shared_region_pager_slid_error;
3334 extern uint64_t shared_region_pager_reclaimed;
3335 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_copied,
3336 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_copied, "");
3337 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_slid,
3338 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_slid, "");
3339 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_slid_error,
3340 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_slid_error, "");
3341 SYSCTL_QUAD(_vm, OID_AUTO, shared_region_pager_reclaimed,
3342 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_reclaimed, "");
3343 extern int shared_region_destroy_delay;
3344 SYSCTL_INT(_vm, OID_AUTO, shared_region_destroy_delay,
3345 CTLFLAG_RW | CTLFLAG_LOCKED, &shared_region_destroy_delay, 0, "");
3346
3347 #if MACH_ASSERT
3348 extern int pmap_ledgers_panic_leeway;
3349 SYSCTL_INT(_vm, OID_AUTO, pmap_ledgers_panic_leeway, CTLFLAG_RW | CTLFLAG_LOCKED, &pmap_ledgers_panic_leeway, 0, "");
3350 #endif /* MACH_ASSERT */
3351
3352
3353 extern uint64_t vm_map_lookup_locked_copy_slowly_count;
3354 extern uint64_t vm_map_lookup_locked_copy_slowly_size;
3355 extern uint64_t vm_map_lookup_locked_copy_slowly_max;
3356 extern uint64_t vm_map_lookup_locked_copy_slowly_restart;
3357 extern uint64_t vm_map_lookup_locked_copy_slowly_error;
3358 extern uint64_t vm_map_lookup_locked_copy_strategically_count;
3359 extern uint64_t vm_map_lookup_locked_copy_strategically_size;
3360 extern uint64_t vm_map_lookup_locked_copy_strategically_max;
3361 extern uint64_t vm_map_lookup_locked_copy_strategically_restart;
3362 extern uint64_t vm_map_lookup_locked_copy_strategically_error;
3363 extern uint64_t vm_map_lookup_locked_copy_shadow_count;
3364 extern uint64_t vm_map_lookup_locked_copy_shadow_size;
3365 extern uint64_t vm_map_lookup_locked_copy_shadow_max;
3366 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_slowly_count,
3367 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_slowly_count, "");
3368 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_slowly_size,
3369 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_slowly_size, "");
3370 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_slowly_max,
3371 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_slowly_max, "");
3372 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_slowly_restart,
3373 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_slowly_restart, "");
3374 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_slowly_error,
3375 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_slowly_error, "");
3376 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_strategically_count,
3377 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_strategically_count, "");
3378 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_strategically_size,
3379 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_strategically_size, "");
3380 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_strategically_max,
3381 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_strategically_max, "");
3382 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_strategically_restart,
3383 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_strategically_restart, "");
3384 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_strategically_error,
3385 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_strategically_error, "");
3386 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_shadow_count,
3387 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_shadow_count, "");
3388 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_shadow_size,
3389 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_shadow_size, "");
3390 SYSCTL_QUAD(_vm, OID_AUTO, map_lookup_locked_copy_shadow_max,
3391 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_map_lookup_locked_copy_shadow_max, "");
3392
3393 extern int vm_protect_privileged_from_untrusted;
3394 SYSCTL_INT(_vm, OID_AUTO, protect_privileged_from_untrusted,
3395 CTLFLAG_RW | CTLFLAG_LOCKED, &vm_protect_privileged_from_untrusted, 0, "");
3396 extern uint64_t vm_copied_on_read;
3397 SYSCTL_QUAD(_vm, OID_AUTO, copied_on_read,
3398 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_copied_on_read, "");
3399
3400 extern int vm_shared_region_count;
3401 extern int vm_shared_region_peak;
3402 SYSCTL_INT(_vm, OID_AUTO, shared_region_count,
3403 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_shared_region_count, 0, "");
3404 SYSCTL_INT(_vm, OID_AUTO, shared_region_peak,
3405 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_shared_region_peak, 0, "");
3406 #if DEVELOPMENT || DEBUG
3407 extern unsigned int shared_region_pagers_resident_count;
3408 SYSCTL_INT(_vm, OID_AUTO, shared_region_pagers_resident_count,
3409 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pagers_resident_count, 0, "");
3410 extern unsigned int shared_region_pagers_resident_peak;
3411 SYSCTL_INT(_vm, OID_AUTO, shared_region_pagers_resident_peak,
3412 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pagers_resident_peak, 0, "");
3413 extern int shared_region_pager_count;
3414 SYSCTL_INT(_vm, OID_AUTO, shared_region_pager_count,
3415 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_pager_count, 0, "");
3416 #if __has_feature(ptrauth_calls)
3417 extern int shared_region_key_count;
3418 SYSCTL_INT(_vm, OID_AUTO, shared_region_key_count,
3419 CTLFLAG_RD | CTLFLAG_LOCKED, &shared_region_key_count, 0, "");
3420 extern int vm_shared_region_reslide_count;
3421 SYSCTL_INT(_vm, OID_AUTO, shared_region_reslide_count,
3422 CTLFLAG_RD | CTLFLAG_LOCKED, &vm_shared_region_reslide_count, 0, "");
3423 #endif /* __has_feature(ptrauth_calls) */
3424 #endif /* DEVELOPMENT || DEBUG */
3425
3426 #if MACH_ASSERT
3427 extern int debug4k_filter;
3428 SYSCTL_INT(_vm, OID_AUTO, debug4k_filter, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_filter, 0, "");
3429 extern int debug4k_panic_on_terminate;
3430 SYSCTL_INT(_vm, OID_AUTO, debug4k_panic_on_terminate, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_panic_on_terminate, 0, "");
3431 extern int debug4k_panic_on_exception;
3432 SYSCTL_INT(_vm, OID_AUTO, debug4k_panic_on_exception, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_panic_on_exception, 0, "");
3433 extern int debug4k_panic_on_misaligned_sharing;
3434 SYSCTL_INT(_vm, OID_AUTO, debug4k_panic_on_misaligned_sharing, CTLFLAG_RW | CTLFLAG_LOCKED, &debug4k_panic_on_misaligned_sharing, 0, "");
3435 #endif /* MACH_ASSERT */
3436
3437 /*
3438 * A sysctl which causes all existing shared regions to become stale. They
3439 * will no longer be used by anything new and will be torn down as soon as
3440 * the last existing user exits. A write of non-zero value causes that to happen.
3441 * This should only be used by launchd, so we check that this is initproc.
3442 */
3443 static int
3444 shared_region_pivot(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
3445 {
3446 unsigned int value = 0;
3447 int changed = 0;
3448 int error = sysctl_io_number(req, 0, sizeof(value), &value, &changed);
3449 if (error || !changed) {
3450 return error;
3451 }
3452 if (current_proc() != initproc) {
3453 return EPERM;
3454 }
3455
3456 vm_shared_region_pivot();
3457
3458 return 0;
3459 }
3460
3461 SYSCTL_PROC(_vm, OID_AUTO, shared_region_pivot,
3462 CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED,
3463 0, 0, shared_region_pivot, "I", "");
3464
3465 /*
3466 * sysctl to return the number of pages on retired_pages_object
3467 */
3468 static int
3469 retired_pages_count SYSCTL_HANDLER_ARGS
3470 {
3471 #pragma unused(arg1, arg2, oidp)
3472 extern uint32_t vm_retired_pages_count(void);
3473 uint32_t value = vm_retired_pages_count();
3474
3475 return SYSCTL_OUT(req, &value, sizeof(value));
3476 }
3477 SYSCTL_PROC(_vm, OID_AUTO, retired_pages_count, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
3478 0, 0, &retired_pages_count, "I", "");
3479
3480 SYSCTL_INT(_vm, OID_AUTO, vmtc_total, CTLFLAG_RD | CTLFLAG_LOCKED,
3481 &vmtc_total, 0, "total text page corruptions detected");
3482 SYSCTL_INT(_vm, OID_AUTO, vmtc_undiagnosed, CTLFLAG_RD | CTLFLAG_LOCKED,
3483 &vmtc_undiagnosed, 0, "undiagnosed text page corruptions");
3484 SYSCTL_INT(_vm, OID_AUTO, vmtc_not_eligible, CTLFLAG_RD | CTLFLAG_LOCKED,
3485 &vmtc_not_eligible, 0, "text page corruptions not eligible for correction");
3486 SYSCTL_INT(_vm, OID_AUTO, vmtc_copyin_fail, CTLFLAG_RD | CTLFLAG_LOCKED,
3487 &vmtc_copyin_fail, 0, "undiagnosed text page corruptions due to copyin failure");
3488 SYSCTL_INT(_vm, OID_AUTO, vmtc_not_found, CTLFLAG_RD | CTLFLAG_LOCKED,
3489 &vmtc_not_found, 0, "text page corruptions but no diff found");
3490 SYSCTL_INT(_vm, OID_AUTO, vmtc_one_bit_flip, CTLFLAG_RD | CTLFLAG_LOCKED,
3491 &vmtc_one_bit_flip, 0, "text page corruptions that had a single bit flip");
3492
3493 SYSCTL_INT(_vm, OID_AUTO, vmtc_1_byte, CTLFLAG_RD | CTLFLAG_LOCKED,
3494 &vmtc_byte_counts[0], 0, "text page corruptions with 1 changed byte");
3495
3496 SYSCTL_INT(_vm, OID_AUTO, vmtc_2_byte, CTLFLAG_RD | CTLFLAG_LOCKED,
3497 &vmtc_byte_counts[1], 0, "text page corruptions with 2 changed bytes");
3498
3499 SYSCTL_INT(_vm, OID_AUTO, vmtc_4_byte, CTLFLAG_RD | CTLFLAG_LOCKED,
3500 &vmtc_byte_counts[2], 0, "text page corruptions with 3 to 4 changed bytes");
3501
3502 SYSCTL_INT(_vm, OID_AUTO, vmtc_8_byte, CTLFLAG_RD | CTLFLAG_LOCKED,
3503 &vmtc_byte_counts[3], 0, "text page corruptions with 5 to 8 changed bytes");
3504
3505 SYSCTL_INT(_vm, OID_AUTO, vmtc_16_byte, CTLFLAG_RD | CTLFLAG_LOCKED,
3506 &vmtc_byte_counts[4], 0, "text page corruptions with 9 to 16 changed bytes");
3507
3508 SYSCTL_INT(_vm, OID_AUTO, vmtc_32_byte, CTLFLAG_RD | CTLFLAG_LOCKED,
3509 &vmtc_byte_counts[5], 0, "text page corruptions with 17 to 32 changed bytes");
3510
3511 SYSCTL_INT(_vm, OID_AUTO, vmtc_64_byte, CTLFLAG_RD | CTLFLAG_LOCKED,
3512 &vmtc_byte_counts[6], 0, "text page corruptions with 33 to 64 changed bytes");
3513
3514 SYSCTL_INT(_vm, OID_AUTO, vmtc_128byte, CTLFLAG_RD | CTLFLAG_LOCKED,
3515 &vmtc_byte_counts[7], 0, "text page corruptions with 65 to 128 changed bytes");
3516
3517 SYSCTL_INT(_vm, OID_AUTO, vmtc_256_byte, CTLFLAG_RD | CTLFLAG_LOCKED,
3518 &vmtc_byte_counts[8], 0, "text page corruptions with >128 changed bytes");
3519
3520 #if DEBUG || DEVELOPMENT
3521 /*
3522 * A sysctl that can be used to corrupt a text page with an illegal instruction.
3523 * Used for testing text page self healing.
3524 */
3525 extern kern_return_t vm_corrupt_text_addr(uintptr_t);
3526 static int
3527 corrupt_text_addr(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
3528 {
3529 uint64_t value = 0;
3530 int error = sysctl_handle_quad(oidp, &value, 0, req);
3531 if (error || !req->newptr) {
3532 return error;
3533 }
3534
3535 if (vm_corrupt_text_addr((uintptr_t)value) == KERN_SUCCESS) {
3536 return 0;
3537 } else {
3538 return EINVAL;
3539 }
3540 }
3541
3542 SYSCTL_PROC(_vm, OID_AUTO, corrupt_text_addr,
3543 CTLTYPE_QUAD | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED,
3544 0, 0, corrupt_text_addr, "-", "");
3545 #endif /* DEBUG || DEVELOPMENT */
mirror of XNU