#include <meta_features.h>
+#include <vm/vm_options.h>
+
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/debug.h>
-#include <kern/lock.h>
#include <kern/extmod_statistics.h>
#include <mach/mach_traps.h>
#include <mach/port.h>
#include <sys/sysctl.h>
#include <sys/cprotect.h>
#include <sys/kpi_socket.h>
+#include <sys/kas_info.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
#include <security/audit/audit.h>
#include <security/mac.h>
#include <vm/vm_protos.h>
-#if CONFIG_FREEZE
#include <sys/kern_memorystatus.h>
-#endif
-int _shared_region_map( struct proc*, int, unsigned int, struct shared_file_mapping_np*, memory_object_control_t*, struct shared_file_mapping_np*);
-int _shared_region_slide(uint32_t, mach_vm_offset_t, mach_vm_size_t, mach_vm_offset_t, mach_vm_size_t, memory_object_control_t);
+int _shared_region_map_and_slide(struct proc*, int, unsigned int, struct shared_file_mapping_np*, uint32_t, user_addr_t, user_addr_t);
int shared_region_copyin_mappings(struct proc*, user_addr_t, unsigned int, struct shared_file_mapping_np *);
-SYSCTL_INT(_vm, OID_AUTO, vm_debug_events, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_debug_events, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_compressor, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_compressor, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_compressor_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_compressor_pages, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_terminate, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_terminate, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, vm_do_collapse_terminate_failure, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.do_collapse_terminate_failure, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, vm_should_cow_but_wired, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.should_cow_but_wired, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_extra_cow, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_extra_cow, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_extra_cow_pages, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_extra_cow_pages, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_lookup_failure_write, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_lookup_failure_write, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, vm_create_upl_lookup_failure_copy, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_counters.create_upl_lookup_failure_copy, 0, "");
+#if VM_SCAN_FOR_SHADOW_CHAIN
+static int vm_shadow_max_enabled = 0; /* Disabled by default */
+extern int proc_shadow_max(void);
+static int
+vm_shadow_max SYSCTL_HANDLER_ARGS
+{
+#pragma unused(arg1, arg2, oidp)
+ int value = 0;
+
+ if (vm_shadow_max_enabled)
+ value = proc_shadow_max();
+ return SYSCTL_OUT(req, &value, sizeof(value));
+}
+SYSCTL_PROC(_vm, OID_AUTO, vm_shadow_max, CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_LOCKED,
+ 0, 0, &vm_shadow_max, "I", "");
+SYSCTL_INT(_vm, OID_AUTO, vm_shadow_max_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_shadow_max_enabled, 0, "");
+
+#endif /* VM_SCAN_FOR_SHADOW_CHAIN */
+
+SYSCTL_INT(_vm, OID_AUTO, vm_debug_events, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_debug_events, 0, "");
+
+__attribute__((noinline)) int __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(
+ mach_port_t task_access_port, int32_t calling_pid, uint32_t calling_gid, int32_t target_pid);
/*
* Sysctl's related to data/stack execution. See osfmk/vm/vm_map.c
*/
int vm_shared_region_unnest_log_interval = 10;
int shared_region_unnest_log_count_threshold = 5;
+/*
+ * Shared cache path enforcement.
+ */
+
+static int scdir_enforce = 1;
+static char scdir_path[] = "/var/db/dyld/";
+
+#ifndef SECURE_KERNEL
+SYSCTL_INT(_vm, OID_AUTO, enforce_shared_cache_dir, CTLFLAG_RW | CTLFLAG_LOCKED, &scdir_enforce, 0, "");
+#endif
+
/* These log rate throttling state variables aren't thread safe, but
* are sufficient unto the task.
*/
user_size_t len,
int prot)
{
+ vm_map_t map;
+
+ map = current_map();
return (vm_map_check_protection(
- current_map(),
- vm_map_trunc_page(addr), vm_map_round_page(addr+len),
+ map,
+ vm_map_trunc_page(addr,
+ vm_map_page_mask(map)),
+ vm_map_round_page(addr+len,
+ vm_map_page_mask(map)),
prot == B_READ ? VM_PROT_READ : VM_PROT_WRITE));
}
user_addr_t addr,
user_size_t len)
{
- kern_return_t kret;
- kret = vm_map_wire(current_map(), vm_map_trunc_page(addr),
- vm_map_round_page(addr+len),
- VM_PROT_READ | VM_PROT_WRITE ,FALSE);
+ kern_return_t kret;
+ vm_map_t map;
+
+ map = current_map();
+ kret = vm_map_wire(map,
+ vm_map_trunc_page(addr,
+ vm_map_page_mask(map)),
+ vm_map_round_page(addr+len,
+ vm_map_page_mask(map)),
+ VM_PROT_READ | VM_PROT_WRITE,
+ FALSE);
switch (kret) {
case KERN_SUCCESS:
vm_map_offset_t vaddr;
ppnum_t paddr;
#endif /* FIXME ] */
- kern_return_t kret;
+ kern_return_t kret;
+ vm_map_t map;
+
+ map = current_map();
#if FIXME /* [ */
if (dirtied) {
pmap = get_task_pmap(current_task());
- for (vaddr = vm_map_trunc_page(addr);
- vaddr < vm_map_round_page(addr+len);
- vaddr += PAGE_SIZE) {
+ for (vaddr = vm_map_trunc_page(addr, PAGE_MASK);
+ vaddr < vm_map_round_page(addr+len, PAGE_MASK);
+ vaddr += PAGE_SIZE) {
paddr = pmap_extract(pmap, vaddr);
pg = PHYS_TO_VM_PAGE(paddr);
vm_page_set_modified(pg);
#ifdef lint
dirtied++;
#endif /* lint */
- kret = vm_map_unwire(current_map(), vm_map_trunc_page(addr),
- vm_map_round_page(addr+len), FALSE);
+ kret = vm_map_unwire(map,
+ vm_map_trunc_page(addr,
+ vm_map_page_mask(map)),
+ vm_map_round_page(addr+len,
+ vm_map_page_mask(map)),
+ FALSE);
switch (kret) {
case KERN_SUCCESS:
return (0);
int allowed;
/* No task_for_pid on bad targets */
- if (target == PROC_NULL || target->p_stat == SZOMB) {
+ if (target->p_stat == SZOMB) {
return FALSE;
}
return allowed;
}
+/*
+ * __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__
+ *
+ * Description: Waits for the user space daemon to respond to the request
+ * we made. Function declared non inline to be visible in
+ * stackshots and spindumps as well as debugging.
+ */
+__attribute__((noinline)) int __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(
+ mach_port_t task_access_port, int32_t calling_pid, uint32_t calling_gid, int32_t target_pid)
+{
+ return check_task_access(task_access_port, calling_pid, calling_gid, target_pid);
+}
+
/*
* Routine: task_for_pid
* Purpose:
p = proc_find(pid);
+ if (p == PROC_NULL) {
+ error = KERN_FAILURE;
+ goto tfpout;
+ }
+
#if CONFIG_AUDIT
- if (p != PROC_NULL)
- AUDIT_ARG(process, p);
+ AUDIT_ARG(process, p);
#endif
if (!(task_for_pid_posix_check(p))) {
}
/* Call up to the task access server */
- error = check_task_access(tfpport, proc_selfpid(), kauth_getgid(), pid);
+ error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
if (error != MACH_MSG_SUCCESS) {
if (error == MACH_RCV_INTERRUPTED)
}
targetproc = proc_find(pid);
+ if (targetproc == PROC_NULL) {
+ error = ESRCH;
+ goto out;
+ }
+
if (!task_for_pid_posix_check(targetproc)) {
error = EPERM;
goto out;
}
target = targetproc->task;
-#ifndef CONFIG_EMBEDDED
if (target != TASK_NULL) {
mach_port_t tfpport;
}
/* Call up to the task access server */
- error = check_task_access(tfpport, proc_selfpid(), kauth_getgid(), pid);
+ error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
if (error != MACH_MSG_SUCCESS) {
if (error == MACH_RCV_INTERRUPTED)
}
}
}
-#endif
task_reference(target);
- error = task_suspend(target);
+ error = task_pidsuspend(target);
if (error) {
if (error == KERN_INVALID_ARGUMENT) {
error = EINVAL;
error = EPERM;
}
}
- task_deallocate(target);
-
-#if CONFIG_FREEZE
- kern_hibernation_on_pid_suspend(pid);
+#if CONFIG_MEMORYSTATUS
+ else {
+ memorystatus_on_suspend(targetproc);
+ }
#endif
+ task_deallocate(target);
+
out:
if (targetproc != PROC_NULL)
proc_rele(targetproc);
}
targetproc = proc_find(pid);
+ if (targetproc == PROC_NULL) {
+ error = ESRCH;
+ goto out;
+ }
+
if (!task_for_pid_posix_check(targetproc)) {
error = EPERM;
goto out;
}
target = targetproc->task;
-#ifndef CONFIG_EMBEDDED
if (target != TASK_NULL) {
mach_port_t tfpport;
}
/* Call up to the task access server */
- error = check_task_access(tfpport, proc_selfpid(), kauth_getgid(), pid);
+ error = __KERNEL_WAITING_ON_TASKGATED_CHECK_ACCESS_UPCALL__(tfpport, proc_selfpid(), kauth_getgid(), pid);
if (error != MACH_MSG_SUCCESS) {
if (error == MACH_RCV_INTERRUPTED)
}
}
}
-#endif
task_reference(target);
-#if CONFIG_FREEZE
- kern_hibernation_on_pid_resume(pid, target);
+#if CONFIG_MEMORYSTATUS
+ memorystatus_on_resume(targetproc);
#endif
- error = task_resume(target);
+ error = task_pidresume(target);
if (error) {
if (error == KERN_INVALID_ARGUMENT) {
error = EINVAL;
} else {
- error = EPERM;
+ if (error == KERN_MEMORY_ERROR) {
+ psignal(targetproc, SIGKILL);
+ error = EIO;
+ } else
+ error = EPERM;
}
}
+
task_deallocate(target);
out:
if (targetproc != PROC_NULL)
proc_rele(targetproc);
- *ret = error;
- return error;
-
- return 0;
-}
-
-#if CONFIG_EMBEDDED
-kern_return_t
-pid_hibernate(struct proc *p __unused, struct pid_hibernate_args *args, int *ret)
-{
- int error = 0;
- proc_t targetproc = PROC_NULL;
- int pid = args->pid;
-
-#ifndef CONFIG_FREEZE
- #pragma unused(pid)
-#else
-
-#if CONFIG_MACF
- error = mac_proc_check_suspend_resume(p, MAC_PROC_CHECK_HIBERNATE);
- if (error) {
- error = EPERM;
- goto out;
- }
-#endif
-
- /*
- * The only accepted pid value here is currently -1, since we just kick off the hibernation thread
- * here - individual ids aren't required. However, it's intended that that this call is to change
- * in the future to initiate hibernation of individual processes. In anticipation, we'll obtain the
- * process handle for potentially valid values and call task_for_pid_posix_check(); this way, everything
- * is validated correctly and set for further refactoring. See <rdar://problem/7839708> for more details.
- */
- if (pid >= 0) {
- targetproc = proc_find(pid);
- if (!task_for_pid_posix_check(targetproc)) {
- error = EPERM;
- goto out;
- }
- }
-
- if (pid == -1) {
- kern_hibernation_on_pid_hibernate(pid);
- } else {
- error = EPERM;
- }
-
-out:
-
-#endif /* CONFIG_FREEZE */
-
- if (targetproc != PROC_NULL)
- proc_rele(targetproc);
+
*ret = error;
return error;
}
-int
-pid_shutdown_sockets(struct proc *p __unused, struct pid_shutdown_sockets_args *args, int *ret)
-{
- int error = 0;
- proc_t targetproc = PROC_NULL;
- struct filedesc *fdp;
- struct fileproc *fp;
- int pid = args->pid;
- int level = args->level;
- int i;
-
- if (level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_SVC &&
- level != SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL)
- {
- error = EINVAL;
- goto out;
- }
-
-#if CONFIG_MACF
- error = mac_proc_check_suspend_resume(p, MAC_PROC_CHECK_SHUTDOWN_SOCKETS);
- if (error) {
- error = EPERM;
- goto out;
- }
-#endif
-
- targetproc = proc_find(pid);
- if (!task_for_pid_posix_check(targetproc)) {
- error = EPERM;
- goto out;
- }
-
- proc_fdlock(targetproc);
- fdp = targetproc->p_fd;
-
- for (i = 0; i < fdp->fd_nfiles; i++) {
- struct socket *sockp;
-
- fp = fdp->fd_ofiles[i];
- if (fp == NULL || (fdp->fd_ofileflags[i] & UF_RESERVED) != 0 ||
- fp->f_fglob->fg_type != DTYPE_SOCKET)
- {
- continue;
- }
-
- sockp = (struct socket *)fp->f_fglob->fg_data;
-
- /* Call networking stack with socket and level */
- (void) socket_defunct(targetproc, sockp, level);
- }
-
- proc_fdunlock(targetproc);
-
-out:
- if (targetproc != PROC_NULL)
- proc_rele(targetproc);
- *ret = error;
- return error;
-}
-#endif /* CONFIG_EMBEDDED */
static int
sysctl_settfp_policy(__unused struct sysctl_oid *oidp, void *arg1,
if (error || req->newptr == USER_ADDR_NULL)
return(error);
- if (!is_suser())
+ if (!kauth_cred_issuser(kauth_cred_get()))
return(EPERM);
if ((error = SYSCTL_IN(req, &new_value, sizeof(int)))) {
__unused int *retvalp)
{
vm_shared_region_t shared_region;
- mach_vm_offset_t start_address;
+ mach_vm_offset_t start_address = 0;
int error;
kern_return_t kr;
SHARED_REGION_TRACE_DEBUG(
("shared_region: %p [%d(%s)] -> check_np(0x%llx)\n",
- current_thread(), p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
(uint64_t)uap->start_address));
/* retrieve the current tasks's shared region */
("shared_region: %p [%d(%s)] "
"check_np(0x%llx) "
"copyout(0x%llx) error %d\n",
- current_thread(), p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
(uint64_t)uap->start_address, (uint64_t)start_address,
error));
}
SHARED_REGION_TRACE_DEBUG(
("shared_region: %p [%d(%s)] check_np(0x%llx) <- 0x%llx %d\n",
- current_thread(), p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
(uint64_t)uap->start_address, (uint64_t)start_address, error));
return error;
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(): "
"copyin(0x%llx, %d) failed (error=%d)\n",
- current_thread(), p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
(uint64_t)user_mappings, mappings_count, error));
}
return error;
* requiring any further setup.
*/
int
-_shared_region_map(
+_shared_region_map_and_slide(
struct proc *p,
int fd,
uint32_t mappings_count,
struct shared_file_mapping_np *mappings,
- memory_object_control_t *sr_file_control,
- struct shared_file_mapping_np *mapping_to_slide)
+ uint32_t slide,
+ user_addr_t slide_start,
+ user_addr_t slide_size)
{
int error;
kern_return_t kr;
struct fileproc *fp;
- struct vnode *vp, *root_vp;
+ struct vnode *vp, *root_vp, *scdir_vp;
struct vnode_attr va;
off_t fs;
memory_object_size_t file_size;
+#if CONFIG_MACF
vm_prot_t maxprot = VM_PROT_ALL;
+#endif
memory_object_control_t file_control;
struct vm_shared_region *shared_region;
SHARED_REGION_TRACE_DEBUG(
("shared_region: %p [%d(%s)] -> map\n",
- current_thread(), p->p_pid, p->p_comm));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm));
shared_region = NULL;
fp = NULL;
vp = NULL;
+ scdir_vp = NULL;
/* get file structure from file descriptor */
error = fp_lookup(p, fd, &fp, 0);
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map: "
"fd=%d lookup failed (error=%d)\n",
- current_thread(), p->p_pid, p->p_comm, fd, error));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm, fd, error));
goto done;
}
/* make sure we're attempting to map a vnode */
- if (fp->f_fglob->fg_type != DTYPE_VNODE) {
+ if (FILEGLOB_DTYPE(fp->f_fglob) != DTYPE_VNODE) {
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map: "
"fd=%d not a vnode (type=%d)\n",
- current_thread(), p->p_pid, p->p_comm,
- fd, fp->f_fglob->fg_type));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ fd, FILEGLOB_DTYPE(fp->f_fglob)));
error = EINVAL;
goto done;
}
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map: "
"fd=%d not readable\n",
- current_thread(), p->p_pid, p->p_comm, fd));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm, fd));
error = EPERM;
goto done;
}
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map: "
"fd=%d getwithref failed (error=%d)\n",
- current_thread(), p->p_pid, p->p_comm, fd, error));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm, fd, error));
goto done;
}
vp = (struct vnode *) fp->f_fglob->fg_data;
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(%p:'%s'): "
"not a file (type=%d)\n",
- current_thread(), p->p_pid, p->p_comm,
- vp, vp->v_name, vp->v_type));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(vp),
+ vp->v_name, vp->v_type));
error = EINVAL;
goto done;
}
#if CONFIG_PROTECT
/* check for content protection access */
{
- void *cnode;
- if ((cnode = cp_get_protected_cnode(vp)) != NULL) {
- error = cp_handle_vnop(cnode, CP_READ_ACCESS | CP_WRITE_ACCESS);
- if (error)
+ error = cp_handle_vnop(vp, CP_READ_ACCESS | CP_WRITE_ACCESS, 0);
+ if (error) {
goto done;
- }
+ }
}
#endif /* CONFIG_PROTECT */
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(%p:'%s'): "
"not on process's root volume\n",
- current_thread(), p->p_pid, p->p_comm,
- vp, vp->v_name));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(vp), vp->v_name));
error = EPERM;
goto done;
}
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(%p:'%s'): "
"vnode_getattr(%p) failed (error=%d)\n",
- current_thread(), p->p_pid, p->p_comm,
- vp, vp->v_name, vp, error));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(vp), vp->v_name,
+ (void *)VM_KERNEL_ADDRPERM(vp), error));
goto done;
}
if (va.va_uid != 0) {
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(%p:'%s'): "
"owned by uid=%d instead of 0\n",
- current_thread(), p->p_pid, p->p_comm,
- vp, vp->v_name, va.va_uid));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(vp),
+ vp->v_name, va.va_uid));
error = EPERM;
goto done;
}
+ if (scdir_enforce) {
+ /* get vnode for scdir_path */
+ error = vnode_lookup(scdir_path, 0, &scdir_vp, vfs_context_current());
+ if (error) {
+ SHARED_REGION_TRACE_ERROR(
+ ("shared_region: %p [%d(%s)] map(%p:'%s'): "
+ "vnode_lookup(%s) failed (error=%d)\n",
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(vp), vp->v_name,
+ scdir_path, error));
+ goto done;
+ }
+
+ /* ensure parent is scdir_vp */
+ if (vnode_parent(vp) != scdir_vp) {
+ SHARED_REGION_TRACE_ERROR(
+ ("shared_region: %p [%d(%s)] map(%p:'%s'): "
+ "shared cache file not in %s\n",
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(vp),
+ vp->v_name, scdir_path));
+ error = EPERM;
+ goto done;
+ }
+ }
+
/* get vnode size */
error = vnode_size(vp, &fs, vfs_context_current());
if (error) {
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(%p:'%s'): "
"vnode_size(%p) failed (error=%d)\n",
- current_thread(), p->p_pid, p->p_comm,
- vp, vp->v_name, vp, error));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(vp), vp->v_name,
+ (void *)VM_KERNEL_ADDRPERM(vp), error));
goto done;
}
file_size = fs;
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(%p:'%s'): "
"no memory object\n",
- current_thread(), p->p_pid, p->p_comm,
- vp, vp->v_name));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(vp), vp->v_name));
error = EINVAL;
goto done;
}
- if (sr_file_control != NULL) {
- *sr_file_control = file_control;
- }
-
-
/* get the process's shared region (setup in vm_map_exec()) */
shared_region = vm_shared_region_get(current_task());
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(%p:'%s'): "
"no shared region\n",
- current_thread(), p->p_pid, p->p_comm,
- vp, vp->v_name));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(vp), vp->v_name));
goto done;
}
file_control,
file_size,
(void *) p->p_fd->fd_rdir,
- mapping_to_slide);
+ slide,
+ slide_start,
+ slide_size);
if (kr != KERN_SUCCESS) {
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(%p:'%s'): "
"vm_shared_region_map_file() failed kr=0x%x\n",
- current_thread(), p->p_pid, p->p_comm,
- vp, vp->v_name, kr));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(vp), vp->v_name, kr));
switch (kr) {
case KERN_INVALID_ADDRESS:
error = EFAULT;
fp_drop(p, fd, fp, 0);
fp = NULL;
}
+ if (scdir_vp != NULL) {
+ (void)vnode_put(scdir_vp);
+ scdir_vp = NULL;
+ }
if (shared_region != NULL) {
vm_shared_region_deallocate(shared_region);
SHARED_REGION_TRACE_DEBUG(
("shared_region: %p [%d(%s)] <- map\n",
- current_thread(), p->p_pid, p->p_comm));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm));
return error;
}
-int
-_shared_region_slide(uint32_t slide,
- mach_vm_offset_t entry_start_address,
- mach_vm_size_t entry_size,
- mach_vm_offset_t slide_start,
- mach_vm_size_t slide_size,
- memory_object_control_t sr_file_control)
-{
- void *slide_info_entry = NULL;
- int error;
-
- if((error = vm_shared_region_slide_init(slide_size, entry_start_address, entry_size, slide, sr_file_control))) {
- printf("slide_info initialization failed with kr=%d\n", error);
- goto done;
- }
-
- slide_info_entry = vm_shared_region_get_slide_info_entry();
- if (slide_info_entry == NULL){
- error = EFAULT;
- } else {
- error = copyin(slide_start,
- slide_info_entry,
- (vm_size_t)slide_size);
- }
- if (error) {
- goto done;
- }
-
- if (vm_shared_region_slide_sanity_check() != KERN_SUCCESS) {
- error = EFAULT;
- printf("Sanity Check failed for slide_info\n");
- } else {
-#if DEBUG
- printf("Succesfully init slide_info with start_address: %p region_size: %ld slide_header_size: %ld\n",
- (void*)(uintptr_t)entry_start_address,
- (unsigned long)entry_size,
- (unsigned long)slide_size);
-#endif
- }
-done:
- return error;
-}
-
int
shared_region_map_and_slide_np(
struct proc *p,
struct shared_region_map_and_slide_np_args *uap,
__unused int *retvalp)
{
- struct shared_file_mapping_np mapping_to_slide;
struct shared_file_mapping_np *mappings;
- unsigned int mappings_count = uap->count;
-
- memory_object_control_t sr_file_control;
+ unsigned int mappings_count = uap->count;
kern_return_t kr = KERN_SUCCESS;
uint32_t slide = uap->slide;
#define SFM_MAX_STACK 8
struct shared_file_mapping_np stack_mappings[SFM_MAX_STACK];
+ /* Is the process chrooted?? */
+ if (p->p_fd->fd_rdir != NULL) {
+ kr = EINVAL;
+ goto done;
+ }
+
if ((kr = vm_shared_region_sliding_valid(slide)) != KERN_SUCCESS) {
if (kr == KERN_INVALID_ARGUMENT) {
/*
* This will happen if we request sliding again
* with the same slide value that was used earlier
- * for the very first sliding. We continue through
- * to the mapping layer. This is so that we can be
- * absolutely certain that the same mappings have
- * been requested.
+ * for the very first sliding.
*/
kr = KERN_SUCCESS;
- } else {
- goto done;
}
+ goto done;
}
if (mappings_count == 0) {
SHARED_REGION_TRACE_INFO(
("shared_region: %p [%d(%s)] map(): "
"no mappings\n",
- current_thread(), p->p_pid, p->p_comm));
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm));
kr = 0; /* no mappings: we're done ! */
goto done;
} else if (mappings_count <= SFM_MAX_STACK) {
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(): "
"too many mappings (%d)\n",
- current_thread(), p->p_pid, p->p_comm,
+ (void *)VM_KERNEL_ADDRPERM(current_thread()),
+ p->p_pid, p->p_comm,
mappings_count));
kr = KERN_FAILURE;
goto done;
}
- kr = _shared_region_map(p, uap->fd, mappings_count, mappings, &sr_file_control, &mapping_to_slide);
+ kr = _shared_region_map_and_slide(p, uap->fd, mappings_count, mappings,
+ slide,
+ uap->slide_start, uap->slide_size);
if (kr != KERN_SUCCESS) {
return kr;
}
- if (slide) {
- kr = _shared_region_slide(slide,
- mapping_to_slide.sfm_file_offset,
- mapping_to_slide.sfm_size,
- uap->slide_start,
- uap->slide_size,
- sr_file_control);
- if (kr != KERN_SUCCESS) {
- vm_shared_region_undo_mappings(NULL, 0, mappings, mappings_count);
- return kr;
- }
- }
done:
return kr;
}
/* sysctl overflow room */
+SYSCTL_INT (_vm, OID_AUTO, pagesize, CTLFLAG_RD | CTLFLAG_LOCKED,
+ (int *) &page_size, 0, "vm page size");
+
/* vm_page_free_target is provided as a makeshift solution for applications that want to
allocate buffer space, possibly purgeable memory, but not cause inactive pages to be
reclaimed. It allows the app to calculate how much memory is free outside the free target. */
SYSCTL_INT(_vm, OID_AUTO, page_purgeable_wired_count, CTLFLAG_RD | CTLFLAG_LOCKED,
&vm_page_purgeable_wired_count, 0, "Wired purgeable page count");
+extern int madvise_free_debug;
+SYSCTL_INT(_vm, OID_AUTO, madvise_free_debug, CTLFLAG_RW | CTLFLAG_LOCKED,
+ &madvise_free_debug, 0, "zero-fill on madvise(MADV_FREE*)");
+
SYSCTL_INT(_vm, OID_AUTO, page_reusable_count, CTLFLAG_RD | CTLFLAG_LOCKED,
&vm_page_stats_reusable.reusable_count, 0, "Reusable page count");
SYSCTL_QUAD(_vm, OID_AUTO, reusable_success, CTLFLAG_RD | CTLFLAG_LOCKED,
&vm_page_stats_reusable.can_reuse_success, "");
SYSCTL_QUAD(_vm, OID_AUTO, can_reuse_failure, CTLFLAG_RD | CTLFLAG_LOCKED,
&vm_page_stats_reusable.can_reuse_failure, "");
+SYSCTL_QUAD(_vm, OID_AUTO, reusable_reclaimed, CTLFLAG_RD | CTLFLAG_LOCKED,
+ &vm_page_stats_reusable.reusable_reclaimed, "");
+
+
+extern unsigned int vm_page_free_count, vm_page_speculative_count;
+SYSCTL_UINT(_vm, OID_AUTO, page_free_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_free_count, 0, "");
+SYSCTL_UINT(_vm, OID_AUTO, page_speculative_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_speculative_count, 0, "");
+
+extern unsigned int vm_page_cleaned_count;
+SYSCTL_UINT(_vm, OID_AUTO, page_cleaned_count, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_page_cleaned_count, 0, "Cleaned queue size");
+
+/* pageout counts */
+extern unsigned int vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external, vm_pageout_inactive_clean, vm_pageout_speculative_clean, vm_pageout_inactive_used;
+extern unsigned int vm_pageout_freed_from_inactive_clean, vm_pageout_freed_from_speculative;
+SYSCTL_UINT(_vm, OID_AUTO, pageout_inactive_dirty_internal, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_inactive_dirty_internal, 0, "");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_inactive_dirty_external, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_inactive_dirty_external, 0, "");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_inactive_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_inactive_clean, 0, "");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_speculative_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_speculative_clean, 0, "");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_inactive_used, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_inactive_used, 0, "");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_freed_from_inactive_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_freed_from_inactive_clean, 0, "");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_freed_from_speculative, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_freed_from_speculative, 0, "");
+
+extern unsigned int vm_pageout_freed_from_cleaned;
+SYSCTL_UINT(_vm, OID_AUTO, pageout_freed_from_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_freed_from_cleaned, 0, "");
+
+/* counts of pages entering the cleaned queue */
+extern unsigned int vm_pageout_enqueued_cleaned, vm_pageout_enqueued_cleaned_from_inactive_clean, vm_pageout_enqueued_cleaned_from_inactive_dirty;
+SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_enqueued_cleaned, 0, ""); /* sum of next two */
+SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned_from_inactive_clean, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_enqueued_cleaned_from_inactive_clean, 0, "");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_enqueued_cleaned_from_inactive_dirty, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_enqueued_cleaned_from_inactive_dirty, 0, "");
+
+/* counts of pages leaving the cleaned queue */
+extern unsigned int vm_pageout_cleaned_reclaimed, vm_pageout_cleaned_reactivated, vm_pageout_cleaned_reference_reactivated, vm_pageout_cleaned_volatile_reactivated, vm_pageout_cleaned_fault_reactivated, vm_pageout_cleaned_commit_reactivated, vm_pageout_cleaned_busy, vm_pageout_cleaned_nolock;
+SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_cleaned_reclaimed, 0, "Cleaned pages reclaimed");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_cleaned_reactivated, 0, "Cleaned pages reactivated"); /* sum of all reactivated AND busy and nolock (even though those actually get reDEactivated */
+SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_reference_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_cleaned_reference_reactivated, 0, "Cleaned pages reference reactivated");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_volatile_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_cleaned_volatile_reactivated, 0, "Cleaned pages volatile reactivated");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_fault_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_cleaned_fault_reactivated, 0, "Cleaned pages fault reactivated");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_commit_reactivated, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_cleaned_commit_reactivated, 0, "Cleaned pages commit reactivated");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_busy, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_cleaned_busy, 0, "Cleaned pages busy (deactivated)");
+SYSCTL_UINT(_vm, OID_AUTO, pageout_cleaned_nolock, CTLFLAG_RD | CTLFLAG_LOCKED, &vm_pageout_cleaned_nolock, 0, "Cleaned pages no-lock (deactivated)");
+
+/* counts of pages prefaulted when entering a memory object */
+extern int64_t vm_prefault_nb_pages, vm_prefault_nb_bailout;
+SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_pages, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_pages, "");
+SYSCTL_QUAD(_vm, OID_AUTO, prefault_nb_bailout, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_prefault_nb_bailout, "");
+
+#include <kern/thread.h>
+#include <sys/user.h>
+void vm_pageout_io_throttle(void);
+
+void vm_pageout_io_throttle(void) {
+ struct uthread *uthread = get_bsdthread_info(current_thread());
+
+ /*
+ * thread is marked as a low priority I/O type
+ * and the I/O we issued while in this cleaning operation
+ * collided with normal I/O operations... we'll
+ * delay in order to mitigate the impact of this
+ * task on the normal operation of the system
+ */
+
+ if (uthread->uu_lowpri_window) {
+ throttle_lowpri_io(1);
+ }
+
+}
int
vm_pressure_monitor(
*retval = (int) pages_wanted;
return 0;
}
+
+int
+kas_info(struct proc *p,
+ struct kas_info_args *uap,
+ int *retval __unused)
+{
+#ifdef SECURE_KERNEL
+ (void)p;
+ (void)uap;
+ return ENOTSUP;
+#else /* !SECURE_KERNEL */
+ int selector = uap->selector;
+ user_addr_t valuep = uap->value;
+ user_addr_t sizep = uap->size;
+ user_size_t size;
+ int error;
+
+ if (!kauth_cred_issuser(kauth_cred_get())) {
+ return EPERM;
+ }
+
+#if CONFIG_MACF
+ error = mac_system_check_kas_info(kauth_cred_get(), selector);
+ if (error) {
+ return error;
+ }
+#endif
+
+ if (IS_64BIT_PROCESS(p)) {
+ user64_size_t size64;
+ error = copyin(sizep, &size64, sizeof(size64));
+ size = (user_size_t)size64;
+ } else {
+ user32_size_t size32;
+ error = copyin(sizep, &size32, sizeof(size32));
+ size = (user_size_t)size32;
+ }
+ if (error) {
+ return error;
+ }
+
+ switch (selector) {
+ case KAS_INFO_KERNEL_TEXT_SLIDE_SELECTOR:
+ {
+ uint64_t slide = vm_kernel_slide;
+
+ if (sizeof(slide) != size) {
+ return EINVAL;
+ }
+
+ if (IS_64BIT_PROCESS(p)) {
+ user64_size_t size64 = (user64_size_t)size;
+ error = copyout(&size64, sizep, sizeof(size64));
+ } else {
+ user32_size_t size32 = (user32_size_t)size;
+ error = copyout(&size32, sizep, sizeof(size32));
+ }
+ if (error) {
+ return error;
+ }
+
+ error = copyout(&slide, valuep, sizeof(slide));
+ if (error) {
+ return error;
+ }
+ }
+ break;
+ default:
+ return EINVAL;
+ }
+
+ return 0;
+#endif /* !SECURE_KERNEL */
+}
projects / apple / xnu.git / blobdiff