2 * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
6 #include <kern/thread.h>
7 #include <kern/assert.h>
8 #include <kern/clock.h>
9 #include <kern/locks.h>
10 #include <kern/sched_prim.h>
11 #include <mach/machine/thread_status.h>
12 #include <mach/thread_act.h>
13 #include <machine/machine_routines.h>
14 #include <arm/thread.h>
15 #include <arm/proc_reg.h>
16 #include <pexpert/pexpert.h>
18 #include <sys/kernel.h>
20 #include <sys/proc_internal.h>
21 #include <sys/syscall.h>
22 #include <sys/systm.h>
24 #include <sys/errno.h>
25 #include <sys/kdebug.h>
26 #include <sys/sysent.h>
27 #include <sys/sysproto.h>
28 #include <sys/kauth.h>
29 #include <sys/bitstring.h>
31 #include <security/audit/audit.h>
34 #include <security/mac_framework.h>
38 extern int32_t dtrace_systrace_syscall(struct proc
*, void *, int *);
39 extern void dtrace_systrace_syscall_return(unsigned short, int, int *);
40 #endif /* CONFIG_DTRACE */
43 unix_syscall(struct arm_saved_state
* regs
, thread_t thread_act
,
44 struct uthread
* uthread
, struct proc
* proc
);
46 static int arm_get_syscall_args(uthread_t
, struct arm_saved_state
*, struct sysent
*);
47 static int arm_get_u32_syscall_args(uthread_t
, arm_saved_state32_t
*, struct sysent
*);
48 static void arm_prepare_u32_syscall_return(struct sysent
*, arm_saved_state_t
*, uthread_t
, int);
49 static void arm_prepare_syscall_return(struct sysent
*, struct arm_saved_state
*, uthread_t
, int);
50 static int arm_get_syscall_number(struct arm_saved_state
*);
51 static void arm_trace_unix_syscall(int, struct arm_saved_state
*);
52 static void arm_clear_syscall_error(struct arm_saved_state
*);
63 #define save_r10 r[10]
64 #define save_r11 r[11]
65 #define save_r12 r[12]
66 #define save_r13 r[13]
69 __XNU_PRIVATE_EXTERN
int do_count_syscalls
= 1;
70 __XNU_PRIVATE_EXTERN
int syscalls_log
[SYS_MAXSYSCALL
];
73 #define code_is_kdebug_trace(code) (((code) == SYS_kdebug_trace) || \
74 ((code) == SYS_kdebug_trace64) || \
75 ((code) == SYS_kdebug_trace_string))
78 * Function: unix_syscall
80 * Inputs: regs - pointer to Process Control Block
85 __attribute__((noreturn
))
89 struct arm_saved_state
* state
,
90 __unused thread_t thread_act
,
91 struct uthread
* uthread
,
96 unsigned short code
, syscode
;
100 assert(is_saved_state32(state
));
103 uthread_reset_proc_refcount(uthread
);
105 code
= arm_get_syscall_number(state
);
107 #define unix_syscall_kprintf(x...) /* kprintf("unix_syscall: " x) */
109 if (kdebug_enable
&& !code_is_kdebug_trace(code
)) {
110 arm_trace_unix_syscall(code
, state
);
113 if ((uthread
->uu_flag
& UT_VFORK
))
114 proc
= current_proc();
116 syscode
= (code
< nsysent
) ? code
: SYS_invalid
;
117 callp
= &sysent
[syscode
];
120 * sy_narg is inaccurate on ARM if a 64 bit parameter is specified. Since user_addr_t
121 * is currently a 32 bit type, this is really a long word count. See rdar://problem/6104668.
123 if (callp
->sy_narg
!= 0) {
124 if (arm_get_syscall_args(uthread
, state
, callp
) != 0) {
125 /* Too many arguments, or something failed */
126 unix_syscall_kprintf("arm_get_syscall_args failed.\n");
127 callp
= &sysent
[SYS_invalid
];
131 uthread
->uu_flag
|= UT_NOTCANCELPT
;
132 uthread
->syscall_code
= code
;
134 uthread
->uu_rval
[0] = 0;
137 * r4 is volatile, if we set it to regs->save_r4 here the child
138 * will have parents r4 after execve
140 uthread
->uu_rval
[1] = 0;
145 * ARM runtime will call cerror if the carry bit is set after a
146 * system call, so clear it here for the common case of success.
148 arm_clear_syscall_error(state
);
151 if (do_count_syscalls
> 0) {
152 syscalls_log
[code
]++;
155 pid
= proc_pid(proc
);
158 uthread
->uu_iocount
= 0;
159 uthread
->uu_vpindex
= 0;
161 unix_syscall_kprintf("code %d (pid %d - %s, tid %lld)\n", code
,
162 pid
, proc
->p_comm
, thread_tid(current_thread()));
165 if (__improbable(proc
->syscall_filter_mask
!= NULL
&& !bitstr_test(proc
->syscall_filter_mask
, syscode
))) {
166 error
= mac_proc_check_syscall_unix(proc
, syscode
);
170 #endif /* CONFIG_MACF */
172 AUDIT_SYSCALL_ENTER(code
, proc
, uthread
);
173 error
= (*(callp
->sy_call
)) (proc
, &uthread
->uu_arg
[0], &(uthread
->uu_rval
[0]));
174 AUDIT_SYSCALL_EXIT(code
, proc
, uthread
, error
);
178 #endif /* CONFIG_MACF */
180 unix_syscall_kprintf("code %d, error %d, results %x, %x (pid %d - %s, tid %lld)\n", code
, error
,
181 uthread
->uu_rval
[0], uthread
->uu_rval
[1],
182 pid
, get_bsdtask_info(current_task()) ? proc
->p_comm
: "unknown" , thread_tid(current_thread()));
185 if (uthread
->uu_iocount
) {
186 printf("system call returned with uu_iocount != 0");
190 uthread
->t_dtrace_errno
= error
;
191 #endif /* CONFIG_DTRACE */
192 #if DEBUG || DEVELOPMENT
193 kern_allocation_name_t
194 prior __assert_only
= thread_set_allocation_name(NULL
);
195 assertf(prior
== NULL
, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior
));
196 #endif /* DEBUG || DEVELOPMENT */
198 arm_prepare_syscall_return(callp
, state
, uthread
, error
);
200 uthread
->uu_flag
&= ~UT_NOTCANCELPT
;
201 uthread
->syscall_code
= 0;
203 if (uthread
->uu_lowpri_window
) {
205 * task is marked as a low priority I/O type
206 * and the I/O we issued while in this system call
207 * collided with normal I/O operations... we'll
208 * delay in order to mitigate the impact of this
209 * task on the normal operation of the system
211 throttle_lowpri_io(1);
213 if (kdebug_enable
&& !code_is_kdebug_trace(code
)) {
214 KDBG_RELEASE(BSDDBG_CODE(DBG_BSD_EXCP_SC
, code
) | DBG_FUNC_END
,
215 error
, uthread
->uu_rval
[0], uthread
->uu_rval
[1], pid
);
219 if (__improbable(uthread_get_proc_refcount(uthread
) != 0)) {
220 panic("system call returned with uu_proc_refcount != 0");
225 thread_exception_return();
230 unix_syscall_return(int error
)
233 struct uthread
*uthread
;
235 struct arm_saved_state
*regs
;
237 struct sysent
*callp
;
239 #define unix_syscall_return_kprintf(x...) /* kprintf("unix_syscall_retur
242 thread_act
= current_thread();
243 proc
= current_proc();
244 uthread
= get_bsdthread_info(thread_act
);
246 regs
= find_user_regs(thread_act
);
247 code
= uthread
->syscall_code
;
248 callp
= (code
>= nsysent
) ? &sysent
[SYS_invalid
] : &sysent
[code
];
251 if (callp
->sy_call
== dtrace_systrace_syscall
)
252 dtrace_systrace_syscall_return( code
, error
, uthread
->uu_rval
);
253 #endif /* CONFIG_DTRACE */
254 #if DEBUG || DEVELOPMENT
255 kern_allocation_name_t
256 prior __assert_only
= thread_set_allocation_name(NULL
);
257 assertf(prior
== NULL
, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior
));
258 #endif /* DEBUG || DEVELOPMENT */
260 AUDIT_SYSCALL_EXIT(code
, proc
, uthread
, error
);
263 * Get index into sysent table
265 arm_prepare_syscall_return(callp
, regs
, uthread
, error
);
267 uthread
->uu_flag
&= ~UT_NOTCANCELPT
;
268 uthread
->syscall_code
= 0;
270 if (uthread
->uu_lowpri_window
) {
272 * task is marked as a low priority I/O type
273 * and the I/O we issued while in this system call
274 * collided with normal I/O operations... we'll
275 * delay in order to mitigate the impact of this
276 * task on the normal operation of the system
278 throttle_lowpri_io(1);
280 if (kdebug_enable
&& !code_is_kdebug_trace(code
)) {
281 KDBG_RELEASE(BSDDBG_CODE(DBG_BSD_EXCP_SC
, code
) | DBG_FUNC_END
,
282 error
, uthread
->uu_rval
[0], uthread
->uu_rval
[1], proc
->p_pid
);
285 thread_exception_return();
290 arm_prepare_u32_syscall_return(struct sysent
*callp
, arm_saved_state_t
*regs
, uthread_t uthread
, int error
)
292 assert(is_saved_state32(regs
));
294 arm_saved_state32_t
*ss32
= saved_state32(regs
);
296 if (error
== ERESTART
) {
298 } else if (error
!= EJUSTRETURN
) {
300 ss32
->save_r0
= error
;
302 /* set the carry bit to execute cerror routine */
303 ss32
->cpsr
|= PSR_CF
;
304 unix_syscall_return_kprintf("error: setting carry to trigger cerror call\n");
305 } else { /* (not error) */
306 switch (callp
->sy_return_type
) {
307 case _SYSCALL_RET_INT_T
:
308 case _SYSCALL_RET_UINT_T
:
309 case _SYSCALL_RET_OFF_T
:
310 case _SYSCALL_RET_ADDR_T
:
311 case _SYSCALL_RET_SIZE_T
:
312 case _SYSCALL_RET_SSIZE_T
:
313 case _SYSCALL_RET_UINT64_T
:
314 ss32
->save_r0
= uthread
->uu_rval
[0];
315 ss32
->save_r1
= uthread
->uu_rval
[1];
317 case _SYSCALL_RET_NONE
:
322 panic("unix_syscall: unknown return type");
327 /* else (error == EJUSTRETURN) { nothing } */
332 arm_trace_u32_unix_syscall(int code
, arm_saved_state32_t
*regs
)
334 bool indirect
= (regs
->save_r12
== 0);
336 KDBG_RELEASE(BSDDBG_CODE(DBG_BSD_EXCP_SC
, code
) | DBG_FUNC_START
,
337 regs
->save_r1
, regs
->save_r2
, regs
->save_r3
, regs
->save_r4
);
339 KDBG_RELEASE(BSDDBG_CODE(DBG_BSD_EXCP_SC
, code
) | DBG_FUNC_START
,
340 regs
->save_r0
, regs
->save_r1
, regs
->save_r2
, regs
->save_r3
);
345 arm_clear_u32_syscall_error(arm_saved_state32_t
*regs
)
347 regs
->cpsr
&= ~PSR_CF
;
353 arm_get_syscall_args(uthread_t uthread
, struct arm_saved_state
*state
, struct sysent
*callp
)
355 assert(is_saved_state32(state
));
356 return arm_get_u32_syscall_args(uthread
, saved_state32(state
), callp
);
359 #if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
361 * For armv7k, the alignment constraints of the ABI mean we don't know how the userspace
362 * arguments are arranged without knowing the the prototype of the syscall. So we use mungers
363 * to marshal the userspace data into the uu_arg. This also means we need the same convention
364 * as mach syscalls. That means we use r8 to pass arguments in the BSD case as well.
367 arm_get_u32_syscall_args(uthread_t uthread
, arm_saved_state32_t
*regs
, struct sysent
*callp
)
371 /* This check is probably not very useful since these both come from build-time */
372 if (callp
->sy_arg_bytes
> sizeof(uthread
->uu_arg
))
375 /* get the munger and use it to marshal in the data from userspace */
376 munger
= callp
->sy_arg_munge32
;
377 if (munger
== NULL
|| (callp
->sy_arg_bytes
== 0))
380 return munger(regs
, uthread
->uu_arg
);
384 * For an AArch32 kernel, where we know that we have only AArch32 userland,
385 * we do not do any munging (which is a little confusing, as it is a contrast
386 * to the i386 kernel, where, like the x86_64 kernel, we always munge
387 * arguments from a 32-bit userland out to 64-bit.
390 arm_get_u32_syscall_args(uthread_t uthread
, arm_saved_state32_t
*regs
, struct sysent
*callp
)
393 int flavor
= (regs
->save_r12
== 0 ? 1 : 0);
395 regparams
= (7 - flavor
); /* Indirect value consumes a register */
397 assert((unsigned) callp
->sy_arg_bytes
<= sizeof (uthread
->uu_arg
));
399 if (callp
->sy_arg_bytes
<= (sizeof(uint32_t) * regparams
)) {
401 * Seven arguments or less are passed in registers.
403 memcpy(&uthread
->uu_arg
[0], ®s
->r
[flavor
], callp
->sy_arg_bytes
);
404 } else if (callp
->sy_arg_bytes
<= sizeof(uthread
->uu_arg
)) {
406 * In this case, we composite - take the first args from registers,
407 * the remainder from the stack (offset by the 7 regs therein).
409 unix_syscall_kprintf("%s: spillover...\n", __FUNCTION__
);
410 memcpy(&uthread
->uu_arg
[0] , ®s
->r
[flavor
], regparams
* sizeof(int));
411 if (copyin((user_addr_t
)regs
->sp
+ 7 * sizeof(int), (int *)&uthread
->uu_arg
[0] + regparams
,
412 (callp
->sy_arg_bytes
- (sizeof(uint32_t) * regparams
))) != 0) {
424 arm_get_syscall_number(struct arm_saved_state
*regs
)
426 if (regs
->save_r12
!= 0) {
427 return regs
->save_r12
;
429 return regs
->save_r0
;
434 arm_prepare_syscall_return(struct sysent
*callp
, struct arm_saved_state
*state
, uthread_t uthread
, int error
)
436 assert(is_saved_state32(state
));
437 arm_prepare_u32_syscall_return(callp
, state
, uthread
, error
);
441 arm_trace_unix_syscall(int code
, struct arm_saved_state
*state
)
443 assert(is_saved_state32(state
));
444 arm_trace_u32_unix_syscall(code
, saved_state32(state
));
448 arm_clear_syscall_error(struct arm_saved_state
* state
)
450 assert(is_saved_state32(state
));
451 arm_clear_u32_syscall_error(saved_state32(state
));
454 #elif defined(__arm64__)
455 static void arm_prepare_u64_syscall_return(struct sysent
*, arm_saved_state_t
*, uthread_t
, int);
456 static int arm_get_u64_syscall_args(uthread_t
, arm_saved_state64_t
*, struct sysent
*);
459 arm_get_syscall_args(uthread_t uthread
, struct arm_saved_state
*state
, struct sysent
*callp
)
461 if (is_saved_state32(state
)) {
462 return arm_get_u32_syscall_args(uthread
, saved_state32(state
), callp
);
464 return arm_get_u64_syscall_args(uthread
, saved_state64(state
), callp
);
469 * 64-bit: all arguments in registers. We're willing to use x9, a temporary
470 * register per the ABI, to pass an argument to the kernel for one case,
471 * an indirect syscall with 8 arguments. No munging required, as all arguments
472 * are in 64-bit wide registers already.
475 arm_get_u64_syscall_args(uthread_t uthread
, arm_saved_state64_t
*regs
, struct sysent
*callp
)
479 #if CONFIG_REQUIRES_U32_MUNGING
483 indirect_offset
= (regs
->x
[ARM64_SYSCALL_CODE_REG_NUM
] == 0) ? 1 : 0;
486 * Everything should fit in registers for now.
488 if (callp
->sy_narg
> (int)(sizeof(uthread
->uu_arg
) / sizeof(uthread
->uu_arg
[0]))) {
492 memcpy(&uthread
->uu_arg
[0], ®s
->x
[indirect_offset
], callp
->sy_narg
* sizeof(uint64_t));
494 #if CONFIG_REQUIRES_U32_MUNGING
496 * The indirect system call interface is vararg based. For armv7k, arm64_32,
497 * and arm64, this means we simply lay the values down on the stack, padded to
498 * a width multiple (4 bytes for armv7k and arm64_32, 8 bytes for arm64).
499 * The arm64(_32) stub for syscall will load this data into the registers and
500 * then trap. This gives us register state that corresponds to what we would
501 * expect from a armv7 task, so in this particular case we need to munge the
504 * TODO: Is there a cleaner way to do this check? What we're actually
505 * interested in is whether the task is arm64_32. We don't appear to guarantee
506 * that uu_proc is populated here, which is why this currently uses the
509 mungerp
= callp
->sy_arg_munge32
;
510 assert(uthread
->uu_thread
);
512 if (indirect_offset
&& !ml_thread_is64bit(uthread
->uu_thread
)) {
513 (*mungerp
)(&uthread
->uu_arg
[0]);
520 * When the kernel is running AArch64, munge arguments from 32-bit
521 * userland out to 64-bit.
523 * flavor == 1 indicates an indirect syscall.
526 arm_get_u32_syscall_args(uthread_t uthread
, arm_saved_state32_t
*regs
, struct sysent
*callp
)
529 #if CONFIG_REQUIRES_U32_MUNGING
532 #error U32 syscalls on ARM64 kernel requires munging
534 int flavor
= (regs
->save_r12
== 0 ? 1 : 0);
536 regparams
= (7 - flavor
); /* Indirect value consumes a register */
538 assert((unsigned) callp
->sy_arg_bytes
<= sizeof (uthread
->uu_arg
));
540 if (callp
->sy_arg_bytes
<= (sizeof(uint32_t) * regparams
)) {
542 * Seven arguments or less are passed in registers.
544 memcpy(&uthread
->uu_arg
[0], ®s
->r
[flavor
], callp
->sy_arg_bytes
);
545 } else if (callp
->sy_arg_bytes
<= sizeof(uthread
->uu_arg
)) {
547 * In this case, we composite - take the first args from registers,
548 * the remainder from the stack (offset by the 7 regs therein).
550 unix_syscall_kprintf("%s: spillover...\n", __FUNCTION__
);
551 memcpy(&uthread
->uu_arg
[0] , ®s
->r
[flavor
], regparams
* sizeof(int));
552 if (copyin((user_addr_t
)regs
->sp
+ 7 * sizeof(int), (int *)&uthread
->uu_arg
[0] + regparams
,
553 (callp
->sy_arg_bytes
- (sizeof(uint32_t) * regparams
))) != 0) {
560 #if CONFIG_REQUIRES_U32_MUNGING
562 mungerp
= callp
->sy_arg_munge32
;
563 if (mungerp
!= NULL
) {
564 (*mungerp
)(&uthread
->uu_arg
[0]);
573 arm_get_syscall_number(struct arm_saved_state
*state
)
575 if (is_saved_state32(state
)) {
576 if (saved_state32(state
)->save_r12
!= 0) {
577 return saved_state32(state
)->save_r12
;
579 return saved_state32(state
)->save_r0
;
582 if (saved_state64(state
)->x
[ARM64_SYSCALL_CODE_REG_NUM
] != 0) {
583 return saved_state64(state
)->x
[ARM64_SYSCALL_CODE_REG_NUM
];
585 return saved_state64(state
)->x
[0];
592 arm_prepare_syscall_return(struct sysent
*callp
, struct arm_saved_state
*state
, uthread_t uthread
, int error
)
594 if (is_saved_state32(state
)) {
595 arm_prepare_u32_syscall_return(callp
, state
, uthread
, error
);
597 arm_prepare_u64_syscall_return(callp
, state
, uthread
, error
);
602 arm_prepare_u64_syscall_return(struct sysent
*callp
, arm_saved_state_t
*regs
, uthread_t uthread
, int error
)
604 assert(is_saved_state64(regs
));
606 arm_saved_state64_t
*ss64
= saved_state64(regs
);
608 if (error
== ERESTART
) {
609 add_saved_state_pc(regs
, -4);
610 } else if (error
!= EJUSTRETURN
) {
615 * Set the carry bit to execute cerror routine.
616 * ARM64_TODO: should we have a separate definition?
617 * The bits are the same.
619 ss64
->cpsr
|= PSR_CF
;
620 unix_syscall_return_kprintf("error: setting carry to trigger cerror call\n");
621 } else { /* (not error) */
622 switch (callp
->sy_return_type
) {
623 case _SYSCALL_RET_INT_T
:
624 ss64
->x
[0] = uthread
->uu_rval
[0];
625 ss64
->x
[1] = uthread
->uu_rval
[1];
627 case _SYSCALL_RET_UINT_T
:
628 ss64
->x
[0] = (u_int
)uthread
->uu_rval
[0];
629 ss64
->x
[1] = (u_int
)uthread
->uu_rval
[1];
631 case _SYSCALL_RET_OFF_T
:
632 case _SYSCALL_RET_ADDR_T
:
633 case _SYSCALL_RET_SIZE_T
:
634 case _SYSCALL_RET_SSIZE_T
:
635 case _SYSCALL_RET_UINT64_T
:
636 ss64
->x
[0] = *((uint64_t *)(&uthread
->uu_rval
[0]));
639 case _SYSCALL_RET_NONE
:
642 panic("unix_syscall: unknown return type");
647 /* else (error == EJUSTRETURN) { nothing } */
652 arm_trace_u64_unix_syscall(int code
, arm_saved_state64_t
*regs
)
654 bool indirect
= (regs
->x
[ARM64_SYSCALL_CODE_REG_NUM
] == 0);
656 KDBG_RELEASE(BSDDBG_CODE(DBG_BSD_EXCP_SC
, code
) | DBG_FUNC_START
,
657 regs
->x
[1], regs
->x
[2], regs
->x
[3], regs
->x
[4]);
659 KDBG_RELEASE(BSDDBG_CODE(DBG_BSD_EXCP_SC
, code
) | DBG_FUNC_START
,
660 regs
->x
[0], regs
->x
[1], regs
->x
[2], regs
->x
[3]);
665 arm_trace_unix_syscall(int code
, struct arm_saved_state
*state
)
667 if (is_saved_state32(state
)) {
668 arm_trace_u32_unix_syscall(code
, saved_state32(state
));
670 arm_trace_u64_unix_syscall(code
, saved_state64(state
));
675 arm_clear_u64_syscall_error(arm_saved_state64_t
*regs
)
678 * ARM64_TODO: should we have a separate definition?
679 * The bits are the same.
681 regs
->cpsr
&= ~PSR_CF
;
685 arm_clear_syscall_error(struct arm_saved_state
* state
)
687 if (is_saved_state32(state
)) {
688 arm_clear_u32_syscall_error(saved_state32(state
));
690 arm_clear_u64_syscall_error(saved_state64(state
));
695 #error Unknown architecture.