2 * Copyright (c) 2007 Apple Inc. All rights reserved.
7 * The contents of this file are subject to the terms of the
8 * Common Development and Distribution License, Version 1.0 only
9 * (the "License"). You may not use this file except in compliance
12 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
13 * or http://www.opensolaris.org/os/licensing.
14 * See the License for the specific language governing permissions
15 * and limitations under the License.
17 * When distributing Covered Code, include this CDDL HEADER in each
18 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
19 * If applicable, add the following below this CDDL HEADER, with the
20 * fields enclosed by brackets "[]" replaced with your own identifying
21 * information: Portions Copyright [yyyy] [name of copyright owner]
26 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
27 * Use is subject to license terms.
31 * #pragma ident "@(#)fasttrap_isa.c 1.19 05/09/14 SMI"
36 #define _KERNEL /* Solaris vs. Darwin */
40 #include <sys/fasttrap_isa.h>
41 #include <sys/fasttrap_impl.h>
42 #include <sys/dtrace.h>
43 #include <sys/dtrace_impl.h>
44 #include <kern/task.h>
46 #include <vm/vm_map.h>
47 #include <mach/mach_vm.h>
48 #include <arm/proc_reg.h>
49 #include <arm/thread.h>
50 #include <arm/caches_internal.h>
52 #include <sys/dtrace_ptss.h>
53 #include <kern/debug.h>
55 #include <pexpert/pexpert.h>
57 extern dtrace_id_t dtrace_probeid_error
;
59 /* Solaris proc_t is the struct. Darwin's proc_t is a pointer to it. */
60 #define proc_t struct proc /* Steer clear of the Darwin typedef for proc_t */
62 extern int dtrace_decode_arm64(uint32_t instr
);
63 extern int dtrace_decode_arm(uint32_t instr
);
64 extern int dtrace_decode_thumb(uint32_t instr
);
67 * Lossless User-Land Tracing on ARM
68 * ---------------------------------
70 * The details here will be fleshed out as more of this is implemented. The
71 * basic design will be the same way as tracing works in x86.
73 * Some ARM specific issues:
75 * We need to patch differently for ARM instructions and Thumb instructions.
76 * When we hit a probe, we check to see if the mode we're currently in is the
77 * same as the mode we're patching for. If not, we remove the tracepoint and
78 * abort. This ARM/Thumb information is pulled in from the arch specific
79 * information in the fasttrap probe.
81 * On ARM, any instruction that uses registers can also use the pc as a
82 * register. This presents problems during emulation because we have copied
83 * the instruction and thus the pc can be different. Currently we've emulated
84 * any instructions that use the pc if they can be used in a return probe.
85 * Eventually we will want to support all instructions that use the pc, but
86 * to do so requires disassembling the instruction and reconstituting it by
87 * substituting a different register.
91 #define THUMB_INSTR(x) (*(uint16_t*) &(x))
93 #define SIGNEXTEND(x,v) ((((int) (x)) << (32-(v))) >> (32-(v)))
94 #define ALIGNADDR(x,v) (((x) >> (v)) << (v))
95 #define GETITSTATE(x) ((((x) >> 8) & 0xFC) | (((x) >> 25) & 0x3))
96 #define ISLASTINIT(x) (((x) & 0xF) == 8)
98 #define SET16(x,w) *((uint16_t*) (x)) = (w)
99 #define SET32(x,w) *((uint32_t*) (x)) = (w)
101 #define IS_ARM32_NOP(x) ((x) == 0xE1A00000)
102 /* Marker for is-enabled probes */
103 #define IS_ARM32_IS_ENABLED(x) ((x) == 0xE0200000)
105 #define IS_ARM64_NOP(x) ((x) == 0xD503201F)
106 /* Marker for is-enabled probes */
107 #define IS_ARM64_IS_ENABLED(x) ((x) == 0xD2800000)
109 #define IS_THUMB32_NOP(x) ((x) == 0x46C0)
110 /* Marker for is-enabled probes */
111 #define IS_THUMB32_IS_ENABLED(x) ((x) == 0x4040)
113 #define ARM_LDM_UF (1 << 23)
114 #define ARM_LDM_PF (1 << 24)
115 #define ARM_LDM_WF (1 << 21)
117 #define ARM_LDR_UF (1 << 23)
118 #define ARM_LDR_BF (1 << 22)
123 /* TODO There were some problems with flushing just the cache line that had been modified.
124 * For now, we'll flush the entire cache, until we figure out how to flush just the patched block.
127 InvalidatePoU_Icache();
131 static int fasttrap_tracepoint_init32 (proc_t
*, fasttrap_tracepoint_t
*, user_addr_t
, fasttrap_probe_type_t
);
132 static int fasttrap_tracepoint_init64 (proc_t
*, fasttrap_tracepoint_t
*, user_addr_t
, fasttrap_probe_type_t
);
135 fasttrap_tracepoint_init(proc_t
*p
, fasttrap_tracepoint_t
*tp
,
136 user_addr_t pc
, fasttrap_probe_type_t type
)
138 if (proc_is64bit(p
)) {
139 return fasttrap_tracepoint_init64(p
, tp
, pc
, type
);
141 return fasttrap_tracepoint_init32(p
, tp
, pc
, type
);
146 fasttrap_tracepoint_init32(proc_t
*p
, fasttrap_tracepoint_t
*tp
,
147 user_addr_t pc
, fasttrap_probe_type_t type
)
153 * Read the instruction at the given address out of the process's
154 * address space. We don't have to worry about a debugger
155 * changing this instruction before we overwrite it with our trap
156 * instruction since P_PR_LOCK is set. Since instructions can span
157 * pages, we potentially read the instruction in two parts. If the
158 * second part fails, we just zero out that part of the instruction.
161 * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey...
164 if (uread(p
, &instr
, 4, pc
) != 0)
167 /* We want &instr to always point to the saved instruction, so just copy the
168 * whole thing When cast to a pointer to a uint16_t, that will give us a
169 * pointer to the first two bytes, which is the thumb instruction.
171 tp
->ftt_instr
= instr
;
173 if (tp
->ftt_fntype
!= FASTTRAP_FN_DONE_INIT
) {
174 switch(tp
->ftt_fntype
) {
175 case FASTTRAP_FN_UNKNOWN
:
176 /* Can't instrument without any information. We can add some heuristics later if necessary. */
179 case FASTTRAP_FN_USDT
:
180 if (IS_ARM32_NOP(instr
) || IS_ARM32_IS_ENABLED(instr
)) {
182 } else if (IS_THUMB32_NOP(THUMB_INSTR(instr
)) || IS_THUMB32_IS_ENABLED(THUMB_INSTR(instr
))) {
185 /* Shouldn't reach here - this means we don't recognize
186 * the instruction at one of the USDT probe locations
190 tp
->ftt_fntype
= FASTTRAP_FN_DONE_INIT
;
193 case FASTTRAP_FN_ARM
:
195 tp
->ftt_fntype
= FASTTRAP_FN_DONE_INIT
;
198 case FASTTRAP_FN_THUMB
:
200 tp
->ftt_fntype
= FASTTRAP_FN_DONE_INIT
;
209 tp
->ftt_type
= dtrace_decode_thumb(instr
);
211 tp
->ftt_type
= dtrace_decode_arm(instr
);
214 if (tp
->ftt_type
== FASTTRAP_T_INV
) {
215 /* This is an instruction we either don't recognize or can't instrument */
216 printf("dtrace: fasttrap init32: Unrecognized instruction: %08x at %08llx\n",
217 (tp
->ftt_thumb
&& dtrace_instr_size(tp
->ftt_instr
,tp
->ftt_thumb
) == 2) ? tp
->ftt_instr1
: instr
, pc
);
226 fasttrap_tracepoint_init64(proc_t
*p
, fasttrap_tracepoint_t
*tp
,
227 user_addr_t pc
, fasttrap_probe_type_t type
)
233 * Read the instruction at the given address out of the process's
234 * address space. We don't have to worry about a debugger
235 * changing this instruction before we overwrite it with our trap
236 * instruction since P_PR_LOCK is set. Since instructions can span
237 * pages, we potentially read the instruction in two parts. If the
238 * second part fails, we just zero out that part of the instruction.
241 * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey...
244 if (uread(p
, &instr
, 4, pc
) != 0)
247 tp
->ftt_instr
= instr
;
248 tp
->ftt_thumb
= 0; /* Always zero on 64bit */
250 if (tp
->ftt_fntype
!= FASTTRAP_FN_DONE_INIT
) {
251 switch(tp
->ftt_fntype
) {
252 case FASTTRAP_FN_UNKNOWN
:
254 * On arm64 there is no distinction between
255 * arm vs. thumb mode instruction types.
257 tp
->ftt_fntype
= FASTTRAP_FN_DONE_INIT
;
260 case FASTTRAP_FN_USDT
:
261 if (IS_ARM64_NOP(instr
) || IS_ARM64_IS_ENABLED(instr
)) {
262 tp
->ftt_fntype
= FASTTRAP_FN_DONE_INIT
;
265 * Shouldn't reach here - this means we don't
266 * recognize the instruction at one of the
267 * USDT probe locations
274 case FASTTRAP_FN_ARM
:
275 case FASTTRAP_FN_THUMB
:
278 * If we get an arm or thumb mode type
279 * then we are clearly in the wrong path.
285 tp
->ftt_type
= dtrace_decode_arm64(instr
);
287 if (tp
->ftt_type
== FASTTRAP_T_ARM64_EXCLUSIVE_MEM
) {
288 kprintf("Detected attempt to place DTrace probe on exclusive memory instruction (pc = 0x%llx); refusing to trace (or exclusive operation could never succeed).\n", pc
);
289 tp
->ftt_type
= FASTTRAP_T_INV
;
293 if (tp
->ftt_type
== FASTTRAP_T_INV
) {
294 /* This is an instruction we either don't recognize or can't instrument */
295 printf("dtrace: fasttrap init64: Unrecognized instruction: %08x at %08llx\n", instr
, pc
);
302 // These are not exported from vm_map.h.
303 extern kern_return_t
vm_map_write_user(vm_map_t map
, void *src_p
, vm_map_address_t dst_addr
, vm_size_t size
);
305 /* Patches the instructions. Almost like uwrite, but need special instructions on ARM to flush the caches. */
307 int patchInst(proc_t
*p
, void *buf
, user_size_t len
, user_addr_t a
)
312 ASSERT(p
->task
!= NULL
);
314 task_t task
= p
->task
;
317 * Grab a reference to the task vm_map_t to make sure
318 * the map isn't pulled out from under us.
320 * Because the proc_lock is not held at all times on all code
321 * paths leading here, it is possible for the proc to have
322 * exited. If the map is null, fail.
324 vm_map_t map
= get_task_map_reference(task
);
326 /* Find the memory permissions. */
327 uint32_t nestingDepth
=999999;
328 vm_region_submap_short_info_data_64_t info
;
329 mach_msg_type_number_t count
= VM_REGION_SUBMAP_SHORT_INFO_COUNT_64
;
330 mach_vm_address_t address
= (mach_vm_address_t
)a
;
331 mach_vm_size_t sizeOfRegion
= (mach_vm_size_t
)len
;
333 ret
= mach_vm_region_recurse(map
, &address
, &sizeOfRegion
, &nestingDepth
, (vm_region_recurse_info_t
)&info
, &count
);
334 if (ret
!= KERN_SUCCESS
)
339 if (!(info
.protection
& VM_PROT_WRITE
)) {
340 /* Save the original protection values for restoration later */
341 reprotect
= info
.protection
;
342 if (info
.max_protection
& VM_PROT_WRITE
) {
343 /* The memory is not currently writable, but can be made writable. */
344 /* Making it both writable and executable at the same time causes warning on embedded */
345 ret
= mach_vm_protect (map
, (mach_vm_offset_t
)a
, (mach_vm_size_t
)len
, 0, (reprotect
& ~VM_PROT_EXECUTE
) | VM_PROT_WRITE
);
348 * The memory is not currently writable, and cannot be made writable. We need to COW this memory.
350 * Strange, we can't just say "reprotect | VM_PROT_COPY", that fails.
352 ret
= mach_vm_protect (map
, (mach_vm_offset_t
)a
, (mach_vm_size_t
)len
, 0, VM_PROT_COPY
| VM_PROT_READ
| VM_PROT_WRITE
);
355 if (ret
!= KERN_SUCCESS
)
359 /* The memory was already writable. */
360 reprotect
= VM_PROT_NONE
;
363 ret
= vm_map_write_user( map
,
370 if (ret
!= KERN_SUCCESS
)
373 if (reprotect
!= VM_PROT_NONE
) {
374 ASSERT(reprotect
& VM_PROT_EXECUTE
);
375 ret
= mach_vm_protect (map
, (mach_vm_offset_t
)a
, (mach_vm_size_t
)len
, 0, reprotect
);
379 vm_map_deallocate(map
);
381 ret
= KERN_TERMINATED
;
387 fasttrap_tracepoint_install(proc_t
*p
, fasttrap_tracepoint_t
*tp
)
389 /* The thumb patch is a 2 byte instruction regardless of the size of the original instruction */
393 if (proc_is64bit(p
)) {
395 instr
= FASTTRAP_ARM64_INSTR
;
398 size
= tp
->ftt_thumb
? 2 : 4;
400 *((uint16_t*) &instr
) = FASTTRAP_THUMB32_INSTR
;
402 instr
= FASTTRAP_ARM32_INSTR
;
406 if (patchInst(p
, &instr
, size
, tp
->ftt_pc
) != 0)
409 tp
->ftt_installed
= 1;
415 fasttrap_tracepoint_remove(proc_t
*p
, fasttrap_tracepoint_t
*tp
)
417 /* The thumb patch is a 2 byte instruction regardless of the size of the original instruction */
421 if (proc_is64bit(p
)) {
423 * Distinguish between read or write failures and a changed
427 if (uread(p
, &instr
, size
, tp
->ftt_pc
) != 0)
430 if (instr
!= FASTTRAP_ARM64_INSTR
)
434 * Distinguish between read or write failures and a changed
437 size
= tp
->ftt_thumb
? 2 : 4;
438 if (uread(p
, &instr
, size
, tp
->ftt_pc
) != 0)
442 if (*((uint16_t*) &instr
) != FASTTRAP_THUMB32_INSTR
)
445 if (instr
!= FASTTRAP_ARM32_INSTR
)
450 if (patchInst(p
, &tp
->ftt_instr
, size
, tp
->ftt_pc
) != 0)
454 tp
->ftt_installed
= 0;
460 fasttrap_return_common(proc_t
*p
, arm_saved_state_t
*regs
, user_addr_t pc
, user_addr_t new_pc
)
462 pid_t pid
= p
->p_pid
;
463 fasttrap_tracepoint_t
*tp
;
464 fasttrap_bucket_t
*bucket
;
468 pid_mtx
= &cpu_core
[CPU
->cpu_id
].cpuc_pid_lock
;
469 lck_mtx_lock(pid_mtx
);
470 bucket
= &fasttrap_tpoints
.fth_table
[FASTTRAP_TPOINTS_INDEX(pid
, pc
)];
472 for (tp
= bucket
->ftb_data
; tp
!= NULL
; tp
= tp
->ftt_next
) {
473 if (pid
== tp
->ftt_pid
&& pc
== tp
->ftt_pc
&&
474 tp
->ftt_proc
->ftpc_acount
!= 0)
479 * Don't sweat it if we can't find the tracepoint again; unlike
480 * when we're in fasttrap_pid_probe(), finding the tracepoint here
481 * is not essential to the correct execution of the process.
484 lck_mtx_unlock(pid_mtx
);
488 for (id
= tp
->ftt_retids
; id
!= NULL
; id
= id
->fti_next
) {
489 fasttrap_probe_t
*probe
= id
->fti_probe
;
491 * If there's a branch that could act as a return site, we
492 * need to trace it, and check here if the program counter is
493 * external to the function.
495 if (is_saved_state32(regs
))
497 if (tp
->ftt_type
!= FASTTRAP_T_LDM_PC
&&
498 tp
->ftt_type
!= FASTTRAP_T_POP_PC
&&
499 new_pc
- probe
->ftp_faddr
< probe
->ftp_fsize
)
503 /* ARM64_TODO - check for FASTTRAP_T_RET */
504 if ((tp
->ftt_type
!= FASTTRAP_T_ARM64_RET
) &&
505 new_pc
- probe
->ftp_faddr
< probe
->ftp_fsize
)
508 if (probe
->ftp_prov
->ftp_provider_type
== DTFTP_PROVIDER_ONESHOT
) {
509 uint8_t already_triggered
= atomic_or_8(&probe
->ftp_triggered
, 1);
510 if (already_triggered
) {
515 * If we have at least one probe associated that
516 * is not a oneshot probe, don't remove the
523 #ifndef CONFIG_EMBEDDED
524 if (ISSET(current_proc()->p_lflag
, P_LNOATTACH
)) {
525 dtrace_probe(dtrace_probeid_error
, 0 /* state */, id
->fti_probe
->ftp_id
,
526 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV
);
531 if (is_saved_state32(regs
)) {
532 dtrace_probe(probe
->ftp_id
,
533 pc
- id
->fti_probe
->ftp_faddr
,
534 saved_state32(regs
)->r
[0], 0, 0, 0);
536 dtrace_probe(probe
->ftp_id
,
537 pc
- id
->fti_probe
->ftp_faddr
,
538 saved_state64(regs
)->x
[0], 0, 0, 0);
543 fasttrap_tracepoint_retire(p
, tp
);
546 lck_mtx_unlock(pid_mtx
);
550 fasttrap_sigsegv(proc_t
*p
, uthread_t t
, user_addr_t addr
, arm_saved_state_t
*regs
)
552 /* TODO: This function isn't implemented yet. In debug mode, panic the system to
553 * find out why we're hitting this point. In other modes, kill the process.
556 #pragma unused(p,t,addr,arm_saved_state)
557 panic("fasttrap: sigsegv not yet implemented");
559 #pragma unused(p,t,addr)
560 /* Kill the process */
561 set_saved_state_pc(regs
, 0);
567 /* Set fault address and mark signal */
569 t
->uu_siglist
|= sigmask(SIGSEGV
);
572 * XXX These two line may be redundant; if not, then we need
573 * XXX to potentially set the data address in the machine
574 * XXX specific thread state structure to indicate the address.
576 t
->uu_exception
= KERN_INVALID_ADDRESS
; /* SIGSEGV */
577 t
->uu_subcode
= 0; /* XXX pad */
582 signal_setast(t
->uu_context
.vc_thread
);
587 fasttrap_usdt_args32(fasttrap_probe_t
*probe
, arm_saved_state32_t
*regs32
, int argc
,
590 int i
, x
, cap
= MIN(argc
, probe
->ftp_nargs
);
592 for (i
= 0; i
< cap
; i
++) {
593 x
= probe
->ftp_argmap
[i
];
595 /* Up to 4 args are passed in registers on arm */
597 argv
[i
] = regs32
->r
[x
];
600 fasttrap_fuword32_noerr(regs32
->sp
+ (x
- 4) * sizeof(uint32_t), &arg
);
606 for (; i
< argc
; i
++) {
612 fasttrap_usdt_args64(fasttrap_probe_t
*probe
, arm_saved_state64_t
*regs64
, int argc
,
615 int i
, x
, cap
= MIN(argc
, probe
->ftp_nargs
);
617 for (i
= 0; i
< cap
; i
++) {
618 x
= probe
->ftp_argmap
[i
];
620 /* Up to 8 args are passed in registers on arm64 */
622 argv
[i
] = regs64
->x
[x
];
624 fasttrap_fuword64_noerr(regs64
->sp
+ (x
- 8) * sizeof(uint64_t), &argv
[i
]);
628 for (; i
< argc
; i
++) {
633 static int condition_true(int cond
, int cpsr
)
636 int zf
= (cpsr
& PSR_ZF
) ? 1 : 0,
637 nf
= (cpsr
& PSR_NF
) ? 1 : 0,
638 cf
= (cpsr
& PSR_CF
) ? 1 : 0,
639 vf
= (cpsr
& PSR_VF
) ? 1 : 0;
642 case 0: taken
= zf
; break;
643 case 1: taken
= !zf
; break;
644 case 2: taken
= cf
; break;
645 case 3: taken
= !cf
; break;
646 case 4: taken
= nf
; break;
647 case 5: taken
= !nf
; break;
648 case 6: taken
= vf
; break;
649 case 7: taken
= !vf
; break;
650 case 8: taken
= (cf
&& !zf
); break;
651 case 9: taken
= (!cf
|| zf
); break;
652 case 10: taken
= (nf
== vf
); break;
653 case 11: taken
= (nf
!= vf
); break;
654 case 12: taken
= (!zf
&& (nf
== vf
)); break;
655 case 13: taken
= (zf
|| (nf
!= vf
)); break;
656 case 14: taken
= 1; break;
657 case 15: taken
= 1; break; /* always "true" for ARM, unpredictable for THUMB. */
663 static void set_thumb_flag(arm_saved_state32_t
*regs32
, user_addr_t pc
)
666 regs32
->cpsr
|= PSR_TF
;
668 regs32
->cpsr
&= ~PSR_TF
;
673 fasttrap_pid_probe_thumb_state_valid(arm_saved_state32_t
*state32
, fasttrap_tracepoint_t
*tp
)
675 uint32_t cpsr
= state32
->cpsr
;
676 uint32_t itstate
= GETITSTATE(cpsr
);
678 /* If in IT block, make sure it's the last statement in the block */
679 if ((itstate
!= 0) && !ISLASTINIT(itstate
)) {
680 printf("dtrace: fasttrap: Tried to trace instruction %08x at %08x but not at end of IT block\n",
681 (tp
->ftt_thumb
&& dtrace_instr_size(tp
->ftt_instr
,tp
->ftt_thumb
) == 2) ? tp
->ftt_instr1
: tp
->ftt_instr
, state32
->pc
);
685 if (!(cpsr
& PSR_TF
)) {
693 fasttrap_get_condition_code(arm_saved_state32_t
*regs32
, fasttrap_tracepoint_t
*tp
)
695 /* Default to always execute */
696 int condition_code
= 0xE;
698 uint32_t itstate
= GETITSTATE(regs32
->cpsr
);
700 /* In IT block, make sure it's the last statement in the block */
701 assert(ISLASTINIT(itstate
));
702 condition_code
= itstate
>> 4;
705 condition_code
= ARM_CONDCODE(tp
->ftt_instr
);
708 return condition_code
;
712 fasttrap_pid_probe_handle_patched_instr32(arm_saved_state_t
*state
, fasttrap_tracepoint_t
*tp
, uthread_t uthread
,
713 proc_t
*p
, uint_t is_enabled
, int *was_simulated
)
715 arm_saved_state32_t
*regs32
= saved_state32(state
);
717 uint32_t pc
= regs32
->pc
;
724 * If there's an is-enabled probe connected to this tracepoint it
725 * means that there was a 'eor r0,r0,r0'
726 * instruction that was placed there by DTrace when the binary was
727 * linked. As this probe is, in fact, enabled, we need to stuff 1
728 * into R0. Accordingly, we can bypass all the instruction
729 * emulation logic since we know the inevitable result. It's possible
730 * that a user could construct a scenario where the 'is-enabled'
731 * probe was on some other instruction, but that would be a rather
732 * exotic way to shoot oneself in the foot.
737 new_pc
= regs32
->pc
+ (tp
->ftt_thumb
? 2 : 4);
741 /* For USDT probes, bypass all the emulation logic for the nop instruction */
742 if ((tp
->ftt_thumb
&& IS_THUMB32_NOP(THUMB_INSTR(tp
->ftt_instr
))) ||
743 (!tp
->ftt_thumb
&& IS_ARM32_NOP(tp
->ftt_instr
))) {
744 new_pc
= regs32
->pc
+ (tp
->ftt_thumb
? 2 : 4);
748 condition_code
= fasttrap_get_condition_code(regs32
, tp
);
749 instr_size
= dtrace_instr_size(tp
->ftt_instr
,tp
->ftt_thumb
);
751 switch (tp
->ftt_type
) {
752 case FASTTRAP_T_MOV_PC_REG
:
753 case FASTTRAP_T_CPY_PC
:
755 if (!condition_true(condition_code
, regs32
->cpsr
)) {
756 new_pc
= pc
+ instr_size
;
762 rm
= THUMB16_HRM(tp
->ftt_instr1
);
764 rm
= tp
->ftt_instr
& 0xF;
766 new_pc
= regs32
->r
[rm
];
768 /* This instruction does not change the Thumb state */
773 case FASTTRAP_T_STM_LR
:
774 case FASTTRAP_T_PUSH_LR
:
777 * This is a very common case, so we want to emulate this instruction if
778 * possible. However, on a push, it is possible that we might reach the end
779 * of a page and have to allocate a new page. Most of the time this will not
780 * happen, and we know that the push instruction can store at most 16 words,
781 * so check to see if we are far from the boundary, and if so, emulate. This
782 * can be made more aggressive by checking the actual number of words being
783 * pushed, but we won't do that for now.
785 * Some of the same issues that apply to POP_PC probably apply here also.
792 if (!condition_true(condition_code
, regs32
->cpsr
)) {
793 new_pc
= pc
+ instr_size
;
798 if (((base
-16*4) >> PAGE_SHIFT
) != (base
>> PAGE_SHIFT
)) {
799 /* Crosses the page boundary, go to emulation */
804 if (instr_size
== 4) {
805 /* We know we have to push lr, never push sp or pc */
806 reglist
= tp
->ftt_instr2
& 0x1FFF;
808 reglist
= tp
->ftt_instr1
& 0xFF;
811 /* We know we have to push lr, never push sp or pc */
812 reglist
= tp
->ftt_instr
& 0x1FFF;
815 /* Push the link register */
817 ret
= fasttrap_suword32(base
, regs32
->lr
);
819 fasttrap_sigsegv(p
, uthread
, (user_addr_t
) base
, state
);
824 /* Start pushing from $r12 */
825 int regmask
= 1 << 12;
829 if (reglist
& regmask
) {
831 ret
= fasttrap_suword32(base
, regs32
->r
[regnum
]);
833 fasttrap_sigsegv(p
, uthread
, (user_addr_t
) base
, state
);
844 new_pc
= pc
+ instr_size
;
850 case FASTTRAP_T_LDM_PC
:
851 case FASTTRAP_T_POP_PC
:
853 /* TODO Two issues that will eventually need to be resolved:
855 * 1. Understand what the hardware does if we have to segfault (data abort) in
856 * the middle of a load multiple. We currently don't have a working segfault
857 * handler anyway, and with no swapfile we should never segfault on this load.
858 * If we do, we'll just kill the process by setting the pc to 0.
860 * 2. The emulation is no longer atomic. We currently only emulate pop for
861 * function epilogues, and so we should never have a race here because one
862 * thread should never be trying to manipulate another thread's stack frames.
863 * That is almost certainly a bug in the program.
865 * This will need to be fixed if we ever:
866 * a. Ship dtrace externally, as this could be a potential attack vector
867 * b. Support instruction level tracing, as we might then pop/ldm non epilogues.
871 /* Assume ldmia! sp/pop ... pc */
873 int regnum
= 0, reglist
;
877 if (!condition_true(condition_code
, regs32
->cpsr
)) {
878 new_pc
= pc
+ instr_size
;
883 if (instr_size
== 4) {
884 /* We know we have to load the pc, don't do it twice */
885 reglist
= tp
->ftt_instr2
& 0x7FFF;
887 reglist
= tp
->ftt_instr1
& 0xFF;
890 /* We know we have to load the pc, don't do it twice */
891 reglist
= tp
->ftt_instr
& 0x7FFF;
897 ret
= fasttrap_fuword32((user_addr_t
)base
, ®s32
->r
[regnum
]);
899 fasttrap_sigsegv(p
, uthread
, (user_addr_t
) base
, state
);
909 ret
= fasttrap_fuword32((user_addr_t
)base
, &new_pc
);
911 fasttrap_sigsegv(p
, uthread
, (user_addr_t
) base
, state
);
919 set_thumb_flag(regs32
, new_pc
);
924 case FASTTRAP_T_CB_N_Z
:
926 /* Thumb mode instruction, and not permitted in IT block, so skip the condition code check */
927 int rn
= tp
->ftt_instr1
& 0x7;
928 int offset
= (((tp
->ftt_instr1
& 0x00F8) >> 2) | ((tp
->ftt_instr1
& 0x0200) >> 3)) + 4;
929 int nonzero
= tp
->ftt_instr1
& 0x0800;
930 if (!nonzero
!= !(regs32
->r
[rn
] == 0)) {
931 new_pc
= pc
+ offset
;
933 new_pc
= pc
+ instr_size
;
938 case FASTTRAP_T_B_COND
:
940 /* Use the condition code in the instruction and ignore the ITSTATE */
944 if (instr_size
== 4) {
945 code
= (tp
->ftt_instr1
>> 6) & 0xF;
946 if (code
== 14 || code
== 15) {
947 panic("fasttrap: Emulation of invalid branch");
949 int S
= (tp
->ftt_instr1
>> 10) & 1,
950 J1
= (tp
->ftt_instr2
>> 13) & 1,
951 J2
= (tp
->ftt_instr2
>> 11) & 1;
952 offset
= 4 + SIGNEXTEND(
953 (S
<< 20) | (J2
<< 19) | (J1
<< 18) |
954 ((tp
->ftt_instr1
& 0x003F) << 12) |
955 ((tp
->ftt_instr2
& 0x07FF) << 1),
958 code
= (tp
->ftt_instr1
>> 8) & 0xF;
959 if (code
== 14 || code
== 15) {
960 panic("fasttrap: Emulation of invalid branch");
962 offset
= 4 + (SIGNEXTEND(tp
->ftt_instr1
& 0xFF, 8) << 1);
965 code
= ARM_CONDCODE(tp
->ftt_instr
);
967 panic("fasttrap: Emulation of invalid branch");
969 offset
= 8 + (SIGNEXTEND(tp
->ftt_instr
& 0x00FFFFFF, 24) << 2);
972 if (condition_true(code
, regs32
->cpsr
)) {
973 new_pc
= pc
+ offset
;
975 new_pc
= pc
+ instr_size
;
981 case FASTTRAP_T_B_UNCOND
:
985 /* Unconditional branches can only be taken from Thumb mode */
986 /* (This is different from an ARM branch with condition code "always") */
987 ASSERT(tp
->ftt_thumb
== 1);
989 if (!condition_true(condition_code
, regs32
->cpsr
)) {
990 new_pc
= pc
+ instr_size
;
994 if (instr_size
== 4) {
995 int S
= (tp
->ftt_instr1
>> 10) & 1,
996 J1
= (tp
->ftt_instr2
>> 13) & 1,
997 J2
= (tp
->ftt_instr2
>> 11) & 1;
998 int I1
= (J1
!= S
) ? 0 : 1, I2
= (J2
!= S
) ? 0 : 1;
999 offset
= 4 + SIGNEXTEND(
1000 (S
<< 24) | (I1
<< 23) | (I2
<< 22) |
1001 ((tp
->ftt_instr1
& 0x03FF) << 12) |
1002 ((tp
->ftt_instr2
& 0x07FF) << 1),
1005 uint32_t instr1
= tp
->ftt_instr1
;
1006 offset
= 4 + (SIGNEXTEND(instr1
& 0x7FF, 11) << 1);
1009 new_pc
= pc
+ offset
;
1014 case FASTTRAP_T_BX_REG
:
1018 if (!condition_true(condition_code
, regs32
->cpsr
)) {
1019 new_pc
= pc
+ instr_size
;
1023 if (tp
->ftt_thumb
) {
1024 reg
= THUMB16_HRM(tp
->ftt_instr1
);
1026 reg
= ARM_RM(tp
->ftt_instr
);
1028 new_pc
= regs32
->r
[reg
];
1029 set_thumb_flag(regs32
, new_pc
);
1034 case FASTTRAP_T_LDR_PC_IMMED
:
1035 case FASTTRAP_T_VLDR_PC_IMMED
:
1036 /* Handle these instructions by replacing the PC in the instruction with another
1037 * register. They are common, so we'd like to support them, and this way we do so
1038 * without any risk of having to simulate a segfault.
1044 case FASTTRAP_T_COMMON
:
1047 uint8_t scratch
[32];
1049 fasttrap_instr_t emul_instr
;
1050 emul_instr
.instr32
= tp
->ftt_instr
;
1051 int emul_instr_size
;
1054 * Unfortunately sometimes when we emulate the instruction and have to replace the
1055 * PC, there is no longer a thumb mode equivalent. We end up having to run the
1056 * modified instruction in ARM mode. We use this variable to keep track of which
1057 * mode we should emulate in. We still use the original variable to determine
1058 * what mode to return to.
1060 uint8_t emul_thumb
= tp
->ftt_thumb
;
1062 uint32_t save_val
= 0;
1065 * Dealing with condition codes and emulation:
1066 * We can't just uniformly do a condition code check here because not all instructions
1067 * have condition codes. We currently do not support an instruction by instruction trace,
1068 * so we can assume that either: 1. We are executing a Thumb instruction, in which case
1069 * we either are not in an IT block and should execute always, or we are last in an IT
1070 * block. Either way, the traced instruction will run correctly, and we won't have any
1071 * problems when we return to the original code, because we will no longer be in the IT
1072 * block. 2. We are executing an ARM instruction, in which case we are ok as long as
1073 * we don't attempt to change the condition code.
1075 if (tp
->ftt_type
== FASTTRAP_T_LDR_PC_IMMED
) {
1076 /* We know we always have a free register (the one we plan to write the
1077 * result value to!). So we'll replace the pc with that one.
1080 if (tp
->ftt_thumb
) {
1081 /* Check to see if thumb or thumb2 */
1082 if (instr_size
== 2) {
1084 * Sadness. We need to emulate this instruction in ARM mode
1085 * because it has an 8 bit immediate offset. Instead of having
1086 * to deal with condition codes in the ARM instruction, we'll
1087 * just check the condition and abort if the condition is false.
1089 if (!condition_true(condition_code
, regs32
->cpsr
)) {
1090 new_pc
= pc
+ instr_size
;
1094 new_reg
= (tp
->ftt_instr1
>> 8) & 0x7;
1095 regs32
->r
[new_reg
] = ALIGNADDR(regs32
->pc
+ 4, 2);
1097 emul_instr
.instr32
= 0xE5900000 | (new_reg
<< 16) | (new_reg
<< 12) | ((tp
->ftt_instr1
& 0xFF) << 2);
1099 /* Thumb2. Just replace the register. */
1100 new_reg
= (tp
->ftt_instr2
>> 12) & 0xF;
1101 regs32
->r
[new_reg
] = ALIGNADDR(regs32
->pc
+ 4, 2);
1102 emul_instr
.instr16
.instr1
&= ~0x000F;
1103 emul_instr
.instr16
.instr1
|= new_reg
;
1106 /* ARM. Just replace the register. */
1107 new_reg
= (tp
->ftt_instr
>> 12) & 0xF;
1108 regs32
->r
[new_reg
] = ALIGNADDR(regs32
->pc
+ 8,2);
1109 emul_instr
.instr32
&= ~0x000F0000;
1110 emul_instr
.instr32
|= new_reg
<< 16;
1112 } else if (tp
->ftt_type
== FASTTRAP_T_VLDR_PC_IMMED
) {
1113 /* This instruction only uses one register, and if we're here, we know
1114 * it must be the pc. So we'll just replace it with R0.
1117 save_val
= regs32
->r
[0];
1118 regs32
->r
[save_reg
] = ALIGNADDR(regs32
->pc
+ (tp
->ftt_thumb
? 4 : 8), 2);
1119 if (tp
->ftt_thumb
) {
1120 emul_instr
.instr16
.instr1
&= ~0x000F;
1122 emul_instr
.instr32
&= ~0x000F0000;
1126 emul_instr_size
= dtrace_instr_size(emul_instr
.instr32
, emul_thumb
);
1130 * tp->ftt_thumb = thumb mode of original instruction
1131 * emul_thumb = thumb mode for emulation
1132 * emul_instr = instruction we are using to emulate original instruction
1133 * emul_instr_size = size of emulating instruction
1136 addr
= uthread
->t_dtrace_scratch
->addr
;
1139 fasttrap_sigtrap(p
, uthread
, pc
); // Should be killing target proc
1144 uthread
->t_dtrace_scrpc
= addr
;
1147 * No way to do an unconditional branch in Thumb mode, shove the address
1148 * onto the user stack and go to the next location with a pop. This can
1149 * segfault if this push happens to cross a stack page, but that's ok, since
1150 * we are running in userland, and the kernel knows how to handle userland
1151 * stack expansions correctly.
1153 * Layout of scratch space for Thumb mode:
1154 * Emulated instruction
1155 * ldr save_reg, [pc, #16] (if necessary, restore any register we clobbered)
1160 * Location we should return to in original program
1161 * Saved value of clobbered register (if necessary)
1164 bcopy(&emul_instr
, &scratch
[i
], emul_instr_size
); i
+= emul_instr_size
;
1166 if (save_reg
!= -1) {
1167 uint16_t restore_inst
= 0x4803;
1168 restore_inst
|= (save_reg
& 0x7) << 8;
1169 SET16(scratch
+i
, restore_inst
); i
+= 2; // ldr reg, [pc , #16]
1172 SET16(scratch
+i
, 0xB403); i
+= 2; // push { r0, r1 }
1173 SET16(scratch
+i
, 0x4801); i
+= 2; // ldr r0, [pc, #4]
1174 SET16(scratch
+i
, 0x9001); i
+= 2; // str r0, [sp, #4]
1175 SET16(scratch
+i
, 0xBD01); i
+= 2; // pop { r0, pc }
1178 SET16(scratch
+i
, 0); i
+= 2; // padding - saved 32 bit words must be aligned
1180 SET32(scratch
+i
, pc
+ instr_size
+ (tp
->ftt_thumb
? 1 : 0)); i
+= 4; // Return address
1181 if (save_reg
!= -1) {
1182 SET32(scratch
+i
, save_val
); i
+= 4; // saved value of clobbered register
1185 uthread
->t_dtrace_astpc
= addr
+ i
;
1186 bcopy(&emul_instr
, &scratch
[i
], emul_instr_size
); i
+= emul_instr_size
;
1187 SET16(scratch
+i
, FASTTRAP_THUMB32_RET_INSTR
); i
+= 2;
1190 * Layout of scratch space for ARM mode:
1191 * Emulated instruction
1192 * ldr save_reg, [pc, #12] (if necessary, restore any register we clobbered)
1194 * Location we should return to in original program
1195 * Saved value of clobbered register (if necessary)
1198 bcopy(&emul_instr
, &scratch
[i
], emul_instr_size
); i
+= emul_instr_size
;
1200 if (save_reg
!= -1) {
1201 uint32_t restore_inst
= 0xE59F0004;
1202 restore_inst
|= save_reg
<< 12;
1203 SET32(scratch
+i
, restore_inst
); i
+= 4; // ldr reg, [pc, #12]
1205 SET32(scratch
+i
, 0xE51FF004); i
+= 4; // ldr pc, [pc, #4]
1207 SET32(scratch
+i
, pc
+ instr_size
+ (tp
->ftt_thumb
? 1 : 0)); i
+= 4; // Return address
1208 if (save_reg
!= -1) {
1209 SET32(scratch
+i
, save_val
); i
+= 4; // Saved value of clobbered register
1212 uthread
->t_dtrace_astpc
= addr
+ i
;
1213 bcopy(&emul_instr
, &scratch
[i
], emul_instr_size
); i
+= emul_instr_size
;
1214 SET32(scratch
+i
, FASTTRAP_ARM32_RET_INSTR
); i
+= 4;
1217 if (patchInst(p
, scratch
, i
, uthread
->t_dtrace_scratch
->write_addr
) != KERN_SUCCESS
) {
1218 fasttrap_sigtrap(p
, uthread
, pc
);
1223 if (tp
->ftt_retids
!= NULL
) {
1224 uthread
->t_dtrace_step
= 1;
1225 uthread
->t_dtrace_ret
= 1;
1226 new_pc
= uthread
->t_dtrace_astpc
+ (emul_thumb
? 1 : 0);
1228 new_pc
= uthread
->t_dtrace_scrpc
+ (emul_thumb
? 1 : 0);
1231 uthread
->t_dtrace_pc
= pc
;
1232 uthread
->t_dtrace_npc
= pc
+ instr_size
;
1233 uthread
->t_dtrace_on
= 1;
1235 set_thumb_flag(regs32
, new_pc
);
1240 panic("fasttrap: mishandled an instruction");
1243 set_saved_state_pc(state
, new_pc
);
1248 * Copy out an instruction for execution in userland.
1249 * Trap back to kernel to handle return to original flow of execution, because
1250 * direct branches don't have sufficient range (+/- 128MB) and we
1251 * cannot clobber a GPR. Note that we have to specially handle PC-rel loads/stores
1252 * as well, which have range +/- 1MB (convert to an indirect load). Instruction buffer
1255 * [ Thunked instruction sequence ]
1256 * [ Trap for return to original code and return probe handling ]
1258 * This *does* make it impossible for an ldxr/stxr pair to succeed if we trace on or between
1259 * them... may need to get fancy at some point.
1262 fasttrap_pid_probe_thunk_instr64(arm_saved_state_t
*state
, fasttrap_tracepoint_t
*tp
, proc_t
*p
, uthread_t uthread
,
1263 const uint32_t *instructions
, uint32_t num_instrs
, user_addr_t
*pc_out
)
1265 uint32_t local_scratch
[8];
1266 user_addr_t pc
= get_saved_state_pc(state
);
1267 user_addr_t user_scratch_area
;
1269 assert(num_instrs
< 8);
1271 bcopy(instructions
, local_scratch
, num_instrs
* sizeof(uint32_t));
1272 local_scratch
[num_instrs
] = FASTTRAP_ARM64_RET_INSTR
;
1274 uthread
->t_dtrace_astpc
= uthread
->t_dtrace_scrpc
= uthread
->t_dtrace_scratch
->addr
;
1275 user_scratch_area
= uthread
->t_dtrace_scratch
->write_addr
;
1277 if (user_scratch_area
== (user_addr_t
)0) {
1278 fasttrap_sigtrap(p
, uthread
, pc
); // Should be killing target proc
1283 if (patchInst(p
, local_scratch
, (num_instrs
+ 1) * sizeof(uint32_t), user_scratch_area
) != KERN_SUCCESS
) {
1284 fasttrap_sigtrap(p
, uthread
, pc
);
1289 /* We're stepping (come back to kernel to adjust PC for return to regular code). */
1290 uthread
->t_dtrace_step
= 1;
1292 /* We may or may not be about to run a return probe (but we wouldn't thunk ret lr)*/
1293 uthread
->t_dtrace_ret
= (tp
->ftt_retids
!= NULL
);
1294 assert(tp
->ftt_type
!= FASTTRAP_T_ARM64_RET
);
1296 /* Set address of instruction we've patched */
1297 uthread
->t_dtrace_pc
= pc
;
1299 /* Any branch would be emulated, next instruction should be one ahead */
1300 uthread
->t_dtrace_npc
= pc
+ 4;
1302 /* We are certainly handling a probe */
1303 uthread
->t_dtrace_on
= 1;
1305 /* Let's jump to the scratch area */
1306 *pc_out
= uthread
->t_dtrace_scratch
->addr
;
1310 * Sign-extend bit "sign_bit_index" out to bit 64.
1313 sign_extend(int64_t input
, uint32_t sign_bit_index
)
1315 assert(sign_bit_index
< 63);
1316 if (input
& (1ULL << sign_bit_index
)) {
1317 /* All 1's & ~[1's from 0 to sign bit] */
1318 input
|= ((~0ULL) & ~((1ULL << (sign_bit_index
+ 1)) - 1ULL));
1325 * Handle xzr vs. sp, fp, lr, etc. Will *not* read the SP.
1328 get_saved_state64_regno(arm_saved_state64_t
*regs64
, uint32_t regno
, int use_xzr
)
1330 /* Set PC to register value */
1344 return regs64
->x
[regno
];
1349 set_saved_state64_regno(arm_saved_state64_t
*regs64
, uint32_t regno
, int use_xzr
, register_t value
)
1351 /* Set PC to register value */
1365 regs64
->x
[regno
] = value
;
1371 * Common operation: extract sign-extended PC offset from instruction
1372 * Left-shifts result by two bits.
1375 extract_address_literal_sign_extended(uint32_t instr
, uint32_t base
, uint32_t numbits
)
1379 offset
= (instr
>> base
) & ((1 << numbits
) - 1);
1380 offset
= sign_extend(offset
, numbits
- 1);
1381 offset
= offset
<< 2;
1387 do_cbz_cnbz(arm_saved_state64_t
*regs64
, uint32_t regwidth
, uint32_t instr
, int is_cbz
, user_addr_t
*pc_out
)
1393 /* Extract register */
1394 regno
= (instr
& 0x1f);
1395 assert(regno
<= 31);
1396 regval
= get_saved_state64_regno(regs64
, regno
, 1);
1398 /* Control for size */
1399 if (regwidth
== 32) {
1400 regval
&= 0xFFFFFFFFULL
;
1403 /* Extract offset */
1404 offset
= extract_address_literal_sign_extended(instr
, 5, 19);
1407 if ((is_cbz
&& regval
== 0) || ((!is_cbz
) && regval
!= 0)) {
1408 /* Set PC from label */
1409 *pc_out
= regs64
->pc
+ offset
;
1412 *pc_out
= regs64
->pc
+ 4;
1417 do_tbz_tbnz(arm_saved_state64_t
*regs64
, uint32_t instr
, int is_tbz
, user_addr_t
*pc_out
)
1419 uint64_t offset
, regval
;
1420 uint32_t bit_index
, b5
, b40
, regno
, bit_set
;
1422 /* Compute offset */
1423 offset
= extract_address_literal_sign_extended(instr
, 5, 14);
1425 /* Extract bit index */
1427 b40
= ((instr
>> 19) & 0x1f);
1428 bit_index
= (b5
<< 5) | b40
;
1429 assert(bit_index
<= 63);
1431 /* Extract register */
1432 regno
= (instr
& 0x1f);
1433 assert(regno
<= 31);
1434 regval
= get_saved_state64_regno(regs64
, regno
, 1);
1437 bit_set
= ((regval
& (1 << bit_index
)) != 0);
1439 if ((is_tbz
&& (!bit_set
)) || ((!is_tbz
) && bit_set
)) {
1440 /* Branch: unsigned addition so overflow defined */
1441 *pc_out
= regs64
->pc
+ offset
;
1444 *pc_out
= regs64
->pc
+ 4;
1450 fasttrap_pid_probe_handle_patched_instr64(arm_saved_state_t
*state
, fasttrap_tracepoint_t
*tp __unused
, uthread_t uthread
,
1451 proc_t
*p
, uint_t is_enabled
, int *was_simulated
)
1454 arm_saved_state64_t
*regs64
= saved_state64(state
);
1455 uint32_t instr
= tp
->ftt_instr
;
1456 user_addr_t new_pc
= 0;
1458 /* Neon state should be threaded throw, but hack it until we have better arm/arm64 integration */
1459 arm_neon_saved_state64_t
*ns64
= &(get_user_neon_regs(uthread
->uu_thread
)->ns_64
);
1461 /* is-enabled probe: set x0 to 1 and step forwards */
1464 set_saved_state_pc(state
, regs64
->pc
+ 4);
1468 /* For USDT probes, bypass all the emulation logic for the nop instruction */
1469 if (IS_ARM64_NOP(tp
->ftt_instr
)) {
1470 set_saved_state_pc(state
, regs64
->pc
+ 4);
1475 /* Only one of many cases in the switch doesn't simulate */
1476 switch(tp
->ftt_type
) {
1478 * Function entry: emulate for speed.
1479 * stp fp, lr, [sp, #-16]!
1481 case FASTTRAP_T_ARM64_STANDARD_FUNCTION_ENTRY
:
1483 /* Store values to stack */
1484 res1
= fasttrap_suword64(regs64
->sp
- 16, regs64
->fp
);
1485 res2
= fasttrap_suword64(regs64
->sp
- 8, regs64
->lr
);
1486 if (res1
!= 0 || res2
!= 0) {
1487 fasttrap_sigsegv(p
, uthread
, regs64
->sp
- (res1
? 16 : 8), state
);
1488 new_pc
= regs64
->pc
; /* Bit of a hack */
1492 /* Move stack pointer */
1495 /* Move PC forward */
1496 new_pc
= regs64
->pc
+ 4;
1502 * PC-relative loads/stores: emulate for correctness.
1503 * All loads are 32bits or greater (no need to handle byte or halfword accesses).
1511 * PRFM label -> becomes a NOP
1513 case FASTTRAP_T_ARM64_LDR_S_PC_REL
:
1514 case FASTTRAP_T_ARM64_LDR_W_PC_REL
:
1515 case FASTTRAP_T_ARM64_LDR_D_PC_REL
:
1516 case FASTTRAP_T_ARM64_LDR_X_PC_REL
:
1517 case FASTTRAP_T_ARM64_LDR_Q_PC_REL
:
1518 case FASTTRAP_T_ARM64_LDRSW_PC_REL
:
1521 uint32_t valsize
, regno
;
1522 user_addr_t address
;
1529 /* Extract 19-bit offset, add to pc */
1530 offset
= extract_address_literal_sign_extended(instr
, 5, 19);
1531 address
= regs64
->pc
+ offset
;
1533 /* Extract destination register */
1534 regno
= (instr
& 0x1f);
1535 assert(regno
<= 31);
1537 /* Read value of desired size from memory */
1538 switch (tp
->ftt_type
) {
1539 case FASTTRAP_T_ARM64_LDR_S_PC_REL
:
1540 case FASTTRAP_T_ARM64_LDR_W_PC_REL
:
1541 case FASTTRAP_T_ARM64_LDRSW_PC_REL
:
1544 case FASTTRAP_T_ARM64_LDR_D_PC_REL
:
1545 case FASTTRAP_T_ARM64_LDR_X_PC_REL
:
1548 case FASTTRAP_T_ARM64_LDR_Q_PC_REL
:
1552 panic("Should never get here!");
1557 if (copyin(address
, &value
, valsize
) != 0) {
1558 fasttrap_sigsegv(p
, uthread
, address
, state
);
1559 new_pc
= regs64
->pc
; /* Bit of a hack, we know about update in fasttrap_sigsegv() */
1563 /* Stash in correct register slot */
1564 switch (tp
->ftt_type
) {
1565 case FASTTRAP_T_ARM64_LDR_W_PC_REL
:
1566 set_saved_state64_regno(regs64
, regno
, 1, value
.val32
);
1568 case FASTTRAP_T_ARM64_LDRSW_PC_REL
:
1569 set_saved_state64_regno(regs64
, regno
, 1, sign_extend(value
.val32
, 31));
1571 case FASTTRAP_T_ARM64_LDR_X_PC_REL
:
1572 set_saved_state64_regno(regs64
, regno
, 1, value
.val64
);
1574 case FASTTRAP_T_ARM64_LDR_S_PC_REL
:
1575 ns64
->v
.s
[regno
][0] = value
.val32
;
1577 case FASTTRAP_T_ARM64_LDR_D_PC_REL
:
1578 ns64
->v
.d
[regno
][0] = value
.val64
;
1580 case FASTTRAP_T_ARM64_LDR_Q_PC_REL
:
1581 ns64
->v
.q
[regno
] = value
.val128
;
1584 panic("Should never get here!");
1588 /* Move PC forward */
1589 new_pc
= regs64
->pc
+ 4;
1595 case FASTTRAP_T_ARM64_PRFM
:
1597 /* Becomes a NOP (architecturally permitted). Just move PC forward */
1598 new_pc
= regs64
->pc
+ 4;
1604 * End explicit memory accesses.
1608 * Branches: parse condition codes if needed, emulate for correctness and
1609 * in the case of the indirect branches, convenience
1615 * TBNZ, Xn|Wn, #uimm16, label
1616 * TBZ, Xn|Wn, #uimm16, label
1625 case FASTTRAP_T_ARM64_B_COND
:
1629 /* Extract condition code */
1630 cond
= (instr
& 0xf);
1632 /* Determine if it passes */
1633 if (condition_true(cond
, regs64
->cpsr
)) {
1636 /* Extract 19-bit target offset, add to PC */
1637 offset
= extract_address_literal_sign_extended(instr
, 5, 19);
1638 new_pc
= regs64
->pc
+ offset
;
1641 new_pc
= regs64
->pc
+ 4;
1648 case FASTTRAP_T_ARM64_CBNZ_W
:
1650 do_cbz_cnbz(regs64
, 32, instr
, 0, &new_pc
);
1654 case FASTTRAP_T_ARM64_CBNZ_X
:
1656 do_cbz_cnbz(regs64
, 64, instr
, 0, &new_pc
);
1660 case FASTTRAP_T_ARM64_CBZ_W
:
1662 do_cbz_cnbz(regs64
, 32, instr
, 1, &new_pc
);
1666 case FASTTRAP_T_ARM64_CBZ_X
:
1668 do_cbz_cnbz(regs64
, 64, instr
, 1, &new_pc
);
1673 case FASTTRAP_T_ARM64_TBNZ
:
1675 do_tbz_tbnz(regs64
, instr
, 0, &new_pc
);
1679 case FASTTRAP_T_ARM64_TBZ
:
1681 do_tbz_tbnz(regs64
, instr
, 1, &new_pc
);
1685 case FASTTRAP_T_ARM64_B
:
1686 case FASTTRAP_T_ARM64_BL
:
1690 /* Extract offset from instruction */
1691 offset
= extract_address_literal_sign_extended(instr
, 0, 26);
1693 /* Update LR if appropriate */
1694 if (tp
->ftt_type
== FASTTRAP_T_ARM64_BL
) {
1695 regs64
->lr
= regs64
->pc
+ 4;
1698 /* Compute PC (unsigned addition for defined overflow) */
1699 new_pc
= regs64
->pc
+ offset
;
1704 case FASTTRAP_T_ARM64_BLR
:
1705 case FASTTRAP_T_ARM64_BR
:
1709 /* Extract register from instruction */
1710 regno
= ((instr
>> 5) & 0x1f);
1711 assert(regno
<= 31);
1713 /* Update LR if appropriate */
1714 if (tp
->ftt_type
== FASTTRAP_T_ARM64_BLR
) {
1715 regs64
->lr
= regs64
->pc
+ 4;
1718 /* Update PC in saved state */
1719 new_pc
= get_saved_state64_regno(regs64
, regno
, 1);
1724 case FASTTRAP_T_ARM64_RET
:
1726 /* Extract register */
1727 unsigned regno
= ((instr
>> 5) & 0x1f);
1728 assert(regno
<= 31);
1730 /* Set PC to register value (xzr, not sp) */
1731 new_pc
= get_saved_state64_regno(regs64
, regno
, 1);
1741 * Address calculations: emulate for correctness.
1746 case FASTTRAP_T_ARM64_ADRP
:
1747 case FASTTRAP_T_ARM64_ADR
:
1749 uint64_t immhi
, immlo
, offset
, result
;
1752 /* Extract destination register */
1753 regno
= (instr
& 0x1f);
1754 assert(regno
<= 31);
1756 /* Extract offset */
1757 immhi
= ((instr
& 0x00ffffe0) >> 5); /* bits [23,5]: 19 bits */
1758 immlo
= ((instr
& 0x60000000) >> 29); /* bits [30,29]: 2 bits */
1760 /* Add to PC. Use unsigned addition so that overflow wraps (rather than being undefined). */
1761 if (tp
->ftt_type
== FASTTRAP_T_ARM64_ADRP
) {
1762 offset
= (immhi
<< 14) | (immlo
<< 12); /* Concatenate bits into [32,12]*/
1763 offset
= sign_extend(offset
, 32); /* Sign extend from bit 32 */
1764 result
= (regs64
->pc
& ~0xfffULL
) + offset
; /* And add to page of current pc */
1766 assert(tp
->ftt_type
== FASTTRAP_T_ARM64_ADR
);
1767 offset
= (immhi
<< 2) | immlo
; /* Concatenate bits into [20,0] */
1768 offset
= sign_extend(offset
, 20); /* Sign-extend */
1769 result
= regs64
->pc
+ offset
; /* And add to page of current pc */
1773 set_saved_state64_regno(regs64
, regno
, 1, result
);
1775 /* Move PC forward */
1776 new_pc
= regs64
->pc
+ 4;
1782 * End address calculations.
1786 * Everything else: thunk to userland
1788 case FASTTRAP_T_COMMON
:
1790 fasttrap_pid_probe_thunk_instr64(state
, tp
, p
, uthread
, &tp
->ftt_instr
, 1, &new_pc
);
1796 panic("An instruction DTrace doesn't expect: %d\n", tp
->ftt_type
);
1801 set_saved_state_pc(state
, new_pc
);
1806 fasttrap_pid_probe(arm_saved_state_t
*state
)
1808 proc_t
*p
= current_proc();
1809 fasttrap_bucket_t
*bucket
;
1811 fasttrap_tracepoint_t
*tp
, tp_local
;
1813 dtrace_icookie_t cookie
;
1814 uint_t is_enabled
= 0;
1815 int was_simulated
, retire_tp
= 1;
1816 int is_64_bit
= is_saved_state64(state
);
1818 uint64_t pc
= get_saved_state_pc(state
);
1820 assert(is_64_bit
|| (pc
<= UINT32_MAX
));
1822 uthread_t uthread
= (uthread_t
) get_bsdthread_info(current_thread());
1825 * It's possible that a user (in a veritable orgy of bad planning)
1826 * could redirect this thread's flow of control before it reached the
1827 * return probe fasttrap. In this case we need to kill the process
1828 * since it's in a unrecoverable state.
1830 if (uthread
->t_dtrace_step
) {
1831 ASSERT(uthread
->t_dtrace_on
);
1832 fasttrap_sigtrap(p
, uthread
, (user_addr_t
)pc
);
1837 * Clear all user tracing flags.
1839 uthread
->t_dtrace_ft
= 0;
1840 uthread
->t_dtrace_pc
= 0;
1841 uthread
->t_dtrace_npc
= 0;
1842 uthread
->t_dtrace_scrpc
= 0;
1843 uthread
->t_dtrace_astpc
= 0;
1844 uthread
->t_dtrace_reg
= 0;
1847 * Treat a child created by a call to vfork(2) as if it were its
1848 * parent. We know that there's only one thread of control in such a
1849 * process: this one.
1851 if (p
->p_lflag
& P_LINVFORK
) {
1853 while (p
->p_lflag
& P_LINVFORK
)
1859 pid_mtx
= &cpu_core
[CPU
->cpu_id
].cpuc_pid_lock
;
1860 lck_mtx_lock(pid_mtx
);
1861 bucket
= &fasttrap_tpoints
.fth_table
[FASTTRAP_TPOINTS_INDEX(pid
,pc
)];
1864 * Lookup the tracepoint that the process just hit.
1866 for (tp
= bucket
->ftb_data
; tp
!= NULL
; tp
= tp
->ftt_next
) {
1867 if (pid
== tp
->ftt_pid
&& pc
== tp
->ftt_pc
&&
1868 tp
->ftt_proc
->ftpc_acount
!= 0)
1873 * If we couldn't find a matching tracepoint, either a tracepoint has
1874 * been inserted without using the pid<pid> ioctl interface (see
1875 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
1878 lck_mtx_unlock(pid_mtx
);
1882 /* Validation of THUMB-related state */
1883 if (tp
->ftt_thumb
) {
1884 if (!fasttrap_pid_probe_thumb_state_valid(saved_state32(state
), tp
)) {
1885 fasttrap_tracepoint_remove(p
, tp
);
1886 lck_mtx_unlock(pid_mtx
);
1891 /* Execute the actual probe */
1892 if (tp
->ftt_ids
!= NULL
) {
1896 if (is_saved_state64(state
)) {
1897 arg4
= get_saved_state_reg(state
, 4);
1900 user_addr_t stack
= (user_addr_t
)get_saved_state_sp(state
);
1902 fasttrap_fuword32_noerr(stack
, &arg
);
1907 /* First four parameters are passed in registers */
1909 for (id
= tp
->ftt_ids
; id
!= NULL
; id
= id
->fti_next
) {
1910 fasttrap_probe_t
*probe
= id
->fti_probe
;
1912 #ifndef CONFIG_EMBEDDED
1913 if (ISSET(current_proc()->p_lflag
, P_LNOATTACH
)) {
1914 dtrace_probe(dtrace_probeid_error
, 0 /* state */, probe
->ftp_id
,
1915 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV
);
1920 if (probe
->ftp_prov
->ftp_provider_type
== DTFTP_PROVIDER_ONESHOT
) {
1921 uint8_t already_triggered
= atomic_or_8(&probe
->ftp_triggered
, 1);
1922 if (already_triggered
) {
1927 * If we have at least one probe associated that
1928 * is not a oneshot probe, don't remove the
1934 if (id
->fti_ptype
== DTFTP_ENTRY
) {
1936 * We note that this was an entry
1937 * probe to help ustack() find the
1940 cookie
= dtrace_interrupt_disable();
1941 DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY
);
1942 dtrace_probe(probe
->ftp_id
,
1943 get_saved_state_reg(state
, 0),
1944 get_saved_state_reg(state
, 1),
1945 get_saved_state_reg(state
, 2),
1946 get_saved_state_reg(state
, 3),
1948 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY
);
1949 dtrace_interrupt_enable(cookie
);
1950 } else if (id
->fti_ptype
== DTFTP_IS_ENABLED
) {
1952 * Note that in this case, we don't
1953 * call dtrace_probe() since it's only
1954 * an artificial probe meant to change
1955 * the flow of control so that it
1956 * encounters the true probe.
1959 } else if (probe
->ftp_argmap
== NULL
) {
1960 dtrace_probe(probe
->ftp_id
,
1961 get_saved_state_reg(state
, 0),
1962 get_saved_state_reg(state
, 1),
1963 get_saved_state_reg(state
, 2),
1964 get_saved_state_reg(state
, 3),
1971 fasttrap_usdt_args64(probe
, saved_state64(state
), 5, t
);
1973 fasttrap_usdt_args32(probe
, saved_state32(state
), 5, t
);
1975 dtrace_probe(probe
->ftp_id
, t
[0], t
[1], t
[2], t
[3], t
[4]);
1980 fasttrap_tracepoint_retire(p
, tp
);
1984 * We're about to do a bunch of work so we cache a local copy of
1985 * the tracepoint to emulate the instruction, and then find the
1986 * tracepoint again later if we need to light up any return probes.
1989 lck_mtx_unlock(pid_mtx
);
1995 * Subroutines should update PC.
1996 * We're setting this earlier than Solaris does, to get a "correct"
1997 * ustack() output. In the Sun code, a() -> b() -> c() -> d() is
1998 * reported at: d, b, a. The new way gives c, b, a, which is closer
1999 * to correct, as the return instruction has already exectued.
2002 fasttrap_pid_probe_handle_patched_instr64(state
, tp
, uthread
, p
, is_enabled
, &was_simulated
);
2004 fasttrap_pid_probe_handle_patched_instr32(state
, tp
, uthread
, p
, is_enabled
, &was_simulated
);
2008 * If there were no return probes when we first found the tracepoint,
2009 * we should feel no obligation to honor any return probes that were
2010 * subsequently enabled -- they'll just have to wait until the next
2013 if (tp
->ftt_retids
!= NULL
) {
2015 * We need to wait until the results of the instruction are
2016 * apparent before invoking any return probes. If this
2017 * instruction was emulated we can just call
2018 * fasttrap_return_common(); if it needs to be executed, we
2019 * need to wait until the user thread returns to the kernel.
2022 * It used to be that only common instructions were simulated.
2023 * For performance reasons, we now simulate some instructions
2024 * when safe and go back to userland otherwise. The was_simulated
2025 * flag means we don't need to go back to userland.
2027 if (was_simulated
) {
2028 fasttrap_return_common(p
, state
, (user_addr_t
)pc
, (user_addr_t
)get_saved_state_pc(state
));
2030 ASSERT(uthread
->t_dtrace_ret
!= 0);
2031 ASSERT(uthread
->t_dtrace_pc
== pc
);
2032 ASSERT(uthread
->t_dtrace_scrpc
!= 0);
2033 ASSERT(((user_addr_t
)get_saved_state_pc(state
)) == uthread
->t_dtrace_astpc
);
2041 fasttrap_return_probe(arm_saved_state_t
*regs
)
2043 proc_t
*p
= current_proc();
2044 uthread_t uthread
= (uthread_t
)get_bsdthread_info(current_thread());
2045 user_addr_t pc
= uthread
->t_dtrace_pc
;
2046 user_addr_t npc
= uthread
->t_dtrace_npc
;
2048 uthread
->t_dtrace_pc
= 0;
2049 uthread
->t_dtrace_npc
= 0;
2050 uthread
->t_dtrace_scrpc
= 0;
2051 uthread
->t_dtrace_astpc
= 0;
2054 * Treat a child created by a call to vfork(2) as if it were its
2055 * parent. We know that there's only one thread of control in such a
2056 * process: this one.
2058 if (p
->p_lflag
& P_LINVFORK
) {
2060 while (p
->p_lflag
& P_LINVFORK
)
2066 * We set rp->r_pc to the address of the traced instruction so
2067 * that it appears to dtrace_probe() that we're on the original
2068 * instruction, and so that the user can't easily detect our
2069 * complex web of lies. dtrace_return_probe() (our caller)
2070 * will correctly set %pc after we return.
2072 set_saved_state_pc(regs
, pc
);
2074 fasttrap_return_common(p
, regs
, pc
, npc
);
2080 fasttrap_pid_getarg(void *arg
, dtrace_id_t id
, void *parg
, int argno
,
2083 #pragma unused(arg, id, parg, aframes)
2084 arm_saved_state_t
* regs
= find_user_regs(current_thread());
2086 if (is_saved_state32(regs
)) {
2087 /* First four arguments are in registers */
2089 return saved_state32(regs
)->r
[argno
];
2091 /* Look on the stack for the rest */
2093 uint32_t* sp
= (uint32_t*)(uintptr_t) saved_state32(regs
)->sp
;
2094 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
2095 value
= dtrace_fuword32((user_addr_t
) (sp
+argno
-4));
2096 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
| CPU_DTRACE_BADADDR
);
2101 /* First eight arguments are in registers */
2103 return saved_state64(regs
)->x
[argno
];
2105 /* Look on the stack for the rest */
2107 uint64_t* sp
= (uint64_t*) saved_state64(regs
)->sp
;
2108 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT
);
2109 value
= dtrace_fuword64((user_addr_t
) (sp
+argno
-8));
2110 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT
| CPU_DTRACE_BADADDR
);
2118 fasttrap_usdt_getarg(void *arg
, dtrace_id_t id
, void *parg
, int argno
, int aframes
)
2120 #pragma unused(arg, id, parg, argno, aframes)
2122 return (fasttrap_anarg(ttolwp(curthread
)->lwp_regs
, 0, argno
));