]> git.saurik.com Git - apple/xnu.git/blob - bsd/dev/i386/fasttrap_isa.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-1228.5.18.tar.gz
[apple/xnu.git] / bsd / dev / i386 / fasttrap_isa.c
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * #pragma ident "@(#)fasttrap_isa.c 1.23 06/09/19 SMI"
29 */
30
31 #ifdef KERNEL
32 #ifndef _KERNEL
33 #define _KERNEL /* Solaris vs. Darwin */
34 #endif
35 #endif
36
37 #include <sys/fasttrap_isa.h>
38 #include <sys/fasttrap_impl.h>
39 #include <sys/dtrace.h>
40 #include <sys/dtrace_impl.h>
41 extern dtrace_id_t dtrace_probeid_error;
42
43 #include "fasttrap_regset.h"
44
45 #include <sys/dtrace_ptss.h>
46 #include <kern/debug.h>
47
48 #define proc_t struct proc
49
50 /*
51 * Lossless User-Land Tracing on x86
52 * ---------------------------------
53 *
54 * The execution of most instructions is not dependent on the address; for
55 * these instructions it is sufficient to copy them into the user process's
56 * address space and execute them. To effectively single-step an instruction
57 * in user-land, we copy out the following sequence of instructions to scratch
58 * space in the user thread's ulwp_t structure.
59 *
60 * We then set the program counter (%eip or %rip) to point to this scratch
61 * space. Once execution resumes, the original instruction is executed and
62 * then control flow is redirected to what was originally the subsequent
63 * instruction. If the kernel attemps to deliver a signal while single-
64 * stepping, the signal is deferred and the program counter is moved into the
65 * second sequence of instructions. The second sequence ends in a trap into
66 * the kernel where the deferred signal is then properly handled and delivered.
67 *
68 * For instructions whose execute is position dependent, we perform simple
69 * emulation. These instructions are limited to control transfer
70 * instructions in 32-bit mode, but in 64-bit mode there's the added wrinkle
71 * of %rip-relative addressing that means that almost any instruction can be
72 * position dependent. For all the details on how we emulate generic
73 * instructions included %rip-relative instructions, see the code in
74 * fasttrap_pid_probe() below where we handle instructions of type
75 * FASTTRAP_T_COMMON (under the header: Generic Instruction Tracing).
76 */
77
78 #define FASTTRAP_MODRM_MOD(modrm) (((modrm) >> 6) & 0x3)
79 #define FASTTRAP_MODRM_REG(modrm) (((modrm) >> 3) & 0x7)
80 #define FASTTRAP_MODRM_RM(modrm) ((modrm) & 0x7)
81 #define FASTTRAP_MODRM(mod, reg, rm) (((mod) << 6) | ((reg) << 3) | (rm))
82
83 #define FASTTRAP_SIB_SCALE(sib) (((sib) >> 6) & 0x3)
84 #define FASTTRAP_SIB_INDEX(sib) (((sib) >> 3) & 0x7)
85 #define FASTTRAP_SIB_BASE(sib) ((sib) & 0x7)
86
87 #define FASTTRAP_REX_W(rex) (((rex) >> 3) & 1)
88 #define FASTTRAP_REX_R(rex) (((rex) >> 2) & 1)
89 #define FASTTRAP_REX_X(rex) (((rex) >> 1) & 1)
90 #define FASTTRAP_REX_B(rex) ((rex) & 1)
91 #define FASTTRAP_REX(w, r, x, b) \
92 (0x40 | ((w) << 3) | ((r) << 2) | ((x) << 1) | (b))
93
94 /*
95 * Single-byte op-codes.
96 */
97 #define FASTTRAP_PUSHL_EBP 0x55
98
99 #define FASTTRAP_JO 0x70
100 #define FASTTRAP_JNO 0x71
101 #define FASTTRAP_JB 0x72
102 #define FASTTRAP_JAE 0x73
103 #define FASTTRAP_JE 0x74
104 #define FASTTRAP_JNE 0x75
105 #define FASTTRAP_JBE 0x76
106 #define FASTTRAP_JA 0x77
107 #define FASTTRAP_JS 0x78
108 #define FASTTRAP_JNS 0x79
109 #define FASTTRAP_JP 0x7a
110 #define FASTTRAP_JNP 0x7b
111 #define FASTTRAP_JL 0x7c
112 #define FASTTRAP_JGE 0x7d
113 #define FASTTRAP_JLE 0x7e
114 #define FASTTRAP_JG 0x7f
115
116 #define FASTTRAP_NOP 0x90
117
118 #define FASTTRAP_MOV_EAX 0xb8
119 #define FASTTRAP_MOV_ECX 0xb9
120
121 #define FASTTRAP_RET16 0xc2
122 #define FASTTRAP_RET 0xc3
123
124 #define FASTTRAP_LOOPNZ 0xe0
125 #define FASTTRAP_LOOPZ 0xe1
126 #define FASTTRAP_LOOP 0xe2
127 #define FASTTRAP_JCXZ 0xe3
128
129 #define FASTTRAP_CALL 0xe8
130 #define FASTTRAP_JMP32 0xe9
131 #define FASTTRAP_JMP8 0xeb
132
133 #define FASTTRAP_INT3 0xcc
134 #define FASTTRAP_INT 0xcd
135 #define T_DTRACE_RET 0x7f
136
137 #define FASTTRAP_2_BYTE_OP 0x0f
138 #define FASTTRAP_GROUP5_OP 0xff
139
140 /*
141 * Two-byte op-codes (second byte only).
142 */
143 #define FASTTRAP_0F_JO 0x80
144 #define FASTTRAP_0F_JNO 0x81
145 #define FASTTRAP_0F_JB 0x82
146 #define FASTTRAP_0F_JAE 0x83
147 #define FASTTRAP_0F_JE 0x84
148 #define FASTTRAP_0F_JNE 0x85
149 #define FASTTRAP_0F_JBE 0x86
150 #define FASTTRAP_0F_JA 0x87
151 #define FASTTRAP_0F_JS 0x88
152 #define FASTTRAP_0F_JNS 0x89
153 #define FASTTRAP_0F_JP 0x8a
154 #define FASTTRAP_0F_JNP 0x8b
155 #define FASTTRAP_0F_JL 0x8c
156 #define FASTTRAP_0F_JGE 0x8d
157 #define FASTTRAP_0F_JLE 0x8e
158 #define FASTTRAP_0F_JG 0x8f
159
160 #define FASTTRAP_EFLAGS_OF 0x800
161 #define FASTTRAP_EFLAGS_DF 0x400
162 #define FASTTRAP_EFLAGS_SF 0x080
163 #define FASTTRAP_EFLAGS_ZF 0x040
164 #define FASTTRAP_EFLAGS_AF 0x010
165 #define FASTTRAP_EFLAGS_PF 0x004
166 #define FASTTRAP_EFLAGS_CF 0x001
167
168 /*
169 * Instruction prefixes.
170 */
171 #define FASTTRAP_PREFIX_OPERAND 0x66
172 #define FASTTRAP_PREFIX_ADDRESS 0x67
173 #define FASTTRAP_PREFIX_CS 0x2E
174 #define FASTTRAP_PREFIX_DS 0x3E
175 #define FASTTRAP_PREFIX_ES 0x26
176 #define FASTTRAP_PREFIX_FS 0x64
177 #define FASTTRAP_PREFIX_GS 0x65
178 #define FASTTRAP_PREFIX_SS 0x36
179 #define FASTTRAP_PREFIX_LOCK 0xF0
180 #define FASTTRAP_PREFIX_REP 0xF3
181 #define FASTTRAP_PREFIX_REPNE 0xF2
182
183 #define FASTTRAP_NOREG 0xff
184
185 /*
186 * Map between instruction register encodings and the kernel constants which
187 * correspond to indicies into struct regs.
188 */
189
190 /*
191 * APPLE NOTE: We are cheating here. The regmap is used to decode which register
192 * a given instruction is trying to reference. OS X does not have extended registers
193 * for 32 bit apps, but the *order* is the same. So for 32 bit state, we will return:
194 *
195 * REG_RAX -> EAX
196 * REG_RCX -> ECX
197 * ...
198 * REG_RDI -> EDI
199 *
200 * The fasttrap_getreg function knows how to make the correct transformation.
201 */
202 #if __sol64 || defined(__APPLE__)
203 static const uint8_t regmap[16] = {
204 REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP, REG_RSI, REG_RDI,
205 REG_R8, REG_R9, REG_R10, REG_R11, REG_R12, REG_R13, REG_R14, REG_R15,
206 };
207 #else
208 static const uint8_t regmap[8] = {
209 EAX, ECX, EDX, EBX, UESP, EBP, ESI, EDI
210 };
211 #endif
212
213 static user_addr_t fasttrap_getreg(x86_saved_state_t *, uint_t);
214
215 static uint64_t
216 fasttrap_anarg(x86_saved_state_t *regs, int function_entry, int argno)
217 {
218 uint64_t value;
219 int shift = function_entry ? 1 : 0;
220
221 x86_saved_state64_t *regs64;
222 x86_saved_state32_t *regs32;
223 unsigned int p_model;
224
225 if (is_saved_state64(regs)) {
226 regs64 = saved_state64(regs);
227 regs32 = NULL;
228 p_model = DATAMODEL_LP64;
229 } else {
230 regs64 = NULL;
231 regs32 = saved_state32(regs);
232 p_model = DATAMODEL_ILP32;
233 }
234
235 if (p_model == DATAMODEL_LP64) {
236 user_addr_t stack;
237
238 /*
239 * In 64-bit mode, the first six arguments are stored in
240 * registers.
241 */
242 if (argno < 6)
243 return ((&regs64->rdi)[argno]);
244
245 stack = regs64->isf.rsp + sizeof(uint64_t) * (argno - 6 + shift);
246 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
247 value = dtrace_fuword64(stack);
248 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR);
249 } else {
250 uint32_t *stack = (uint32_t *)regs32->uesp;
251 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
252 value = dtrace_fuword32((user_addr_t)(unsigned long)&stack[argno + shift]);
253 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR);
254 }
255
256 return (value);
257 }
258
259 /*ARGSUSED*/
260 int
261 fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, user_addr_t pc,
262 fasttrap_probe_type_t type)
263 {
264 #pragma unused(type)
265 uint8_t instr[FASTTRAP_MAX_INSTR_SIZE + 10];
266 size_t len = FASTTRAP_MAX_INSTR_SIZE;
267 size_t first = MIN(len, PAGE_SIZE - (pc & PAGE_MASK));
268 uint_t start = 0;
269 size_t size;
270 int rmindex;
271 uint8_t seg, rex = 0;
272 unsigned int p_model = (p->p_flag & P_LP64) ? DATAMODEL_LP64 : DATAMODEL_ILP32;
273
274 /*
275 * Read the instruction at the given address out of the process's
276 * address space. We don't have to worry about a debugger
277 * changing this instruction before we overwrite it with our trap
278 * instruction since P_PR_LOCK is set. Since instructions can span
279 * pages, we potentially read the instruction in two parts. If the
280 * second part fails, we just zero out that part of the instruction.
281 */
282 /*
283 * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey...
284 */
285 if (uread(p, &instr[0], first, pc) != 0)
286 return (-1);
287 if (len > first &&
288 uread(p, &instr[first], len - first, pc + first) != 0) {
289 bzero(&instr[first], len - first);
290 len = first;
291 }
292
293 /*
294 * If the disassembly fails, then we have a malformed instruction.
295 */
296 if ((size = dtrace_instr_size_isa(instr, p_model, &rmindex)) <= 0)
297 return (-1);
298
299 /*
300 * Make sure the disassembler isn't completely broken.
301 */
302 ASSERT(-1 <= rmindex && rmindex < (int)size);
303
304 /*
305 * If the computed size is greater than the number of bytes read,
306 * then it was a malformed instruction possibly because it fell on a
307 * page boundary and the subsequent page was missing or because of
308 * some malicious user.
309 */
310 if (size > len)
311 return (-1);
312
313 tp->ftt_size = (uint8_t)size;
314 tp->ftt_segment = FASTTRAP_SEG_NONE;
315
316 /*
317 * Find the start of the instruction's opcode by processing any
318 * legacy prefixes.
319 */
320 for (;;) {
321 seg = 0;
322 switch (instr[start]) {
323 case FASTTRAP_PREFIX_SS:
324 seg++;
325 /*FALLTHRU*/
326 case FASTTRAP_PREFIX_GS:
327 seg++;
328 /*FALLTHRU*/
329 case FASTTRAP_PREFIX_FS:
330 seg++;
331 /*FALLTHRU*/
332 case FASTTRAP_PREFIX_ES:
333 seg++;
334 /*FALLTHRU*/
335 case FASTTRAP_PREFIX_DS:
336 seg++;
337 /*FALLTHRU*/
338 case FASTTRAP_PREFIX_CS:
339 seg++;
340 /*FALLTHRU*/
341 case FASTTRAP_PREFIX_OPERAND:
342 case FASTTRAP_PREFIX_ADDRESS:
343 case FASTTRAP_PREFIX_LOCK:
344 case FASTTRAP_PREFIX_REP:
345 case FASTTRAP_PREFIX_REPNE:
346 if (seg != 0) {
347 /*
348 * It's illegal for an instruction to specify
349 * two segment prefixes -- give up on this
350 * illegal instruction.
351 */
352 if (tp->ftt_segment != FASTTRAP_SEG_NONE)
353 return (-1);
354
355 tp->ftt_segment = seg;
356 }
357 start++;
358 continue;
359 }
360 break;
361 }
362
363 #if __sol64 || defined(__APPLE__)
364 /*
365 * Identify the REX prefix on 64-bit processes.
366 */
367 if (p_model == DATAMODEL_LP64 && (instr[start] & 0xf0) == 0x40)
368 rex = instr[start++];
369 #endif
370
371 /*
372 * Now that we're pretty sure that the instruction is okay, copy the
373 * valid part to the tracepoint.
374 */
375 bcopy(instr, tp->ftt_instr, FASTTRAP_MAX_INSTR_SIZE);
376
377 tp->ftt_type = FASTTRAP_T_COMMON;
378 if (instr[start] == FASTTRAP_2_BYTE_OP) {
379 switch (instr[start + 1]) {
380 case FASTTRAP_0F_JO:
381 case FASTTRAP_0F_JNO:
382 case FASTTRAP_0F_JB:
383 case FASTTRAP_0F_JAE:
384 case FASTTRAP_0F_JE:
385 case FASTTRAP_0F_JNE:
386 case FASTTRAP_0F_JBE:
387 case FASTTRAP_0F_JA:
388 case FASTTRAP_0F_JS:
389 case FASTTRAP_0F_JNS:
390 case FASTTRAP_0F_JP:
391 case FASTTRAP_0F_JNP:
392 case FASTTRAP_0F_JL:
393 case FASTTRAP_0F_JGE:
394 case FASTTRAP_0F_JLE:
395 case FASTTRAP_0F_JG:
396 tp->ftt_type = FASTTRAP_T_JCC;
397 tp->ftt_code = (instr[start + 1] & 0x0f) | FASTTRAP_JO;
398 tp->ftt_dest = pc + tp->ftt_size +
399 *(int32_t *)&instr[start + 2];
400 break;
401 }
402 } else if (instr[start] == FASTTRAP_GROUP5_OP) {
403 uint_t mod = FASTTRAP_MODRM_MOD(instr[start + 1]);
404 uint_t reg = FASTTRAP_MODRM_REG(instr[start + 1]);
405 uint_t rm = FASTTRAP_MODRM_RM(instr[start + 1]);
406
407 if (reg == 2 || reg == 4) {
408 uint_t i, sz;
409
410 if (reg == 2)
411 tp->ftt_type = FASTTRAP_T_CALL;
412 else
413 tp->ftt_type = FASTTRAP_T_JMP;
414
415 if (mod == 3)
416 tp->ftt_code = 2;
417 else
418 tp->ftt_code = 1;
419
420 ASSERT(p_model == DATAMODEL_LP64 || rex == 0);
421
422 /*
423 * See AMD x86-64 Architecture Programmer's Manual
424 * Volume 3, Section 1.2.7, Table 1-12, and
425 * Appendix A.3.1, Table A-15.
426 */
427 if (mod != 3 && rm == 4) {
428 uint8_t sib = instr[start + 2];
429 uint_t index = FASTTRAP_SIB_INDEX(sib);
430 uint_t base = FASTTRAP_SIB_BASE(sib);
431
432 tp->ftt_scale = FASTTRAP_SIB_SCALE(sib);
433
434 tp->ftt_index = (index == 4) ?
435 FASTTRAP_NOREG :
436 regmap[index | (FASTTRAP_REX_X(rex) << 3)];
437 tp->ftt_base = (mod == 0 && base == 5) ?
438 FASTTRAP_NOREG :
439 regmap[base | (FASTTRAP_REX_B(rex) << 3)];
440
441 i = 3;
442 sz = mod == 1 ? 1 : 4;
443 } else {
444 /*
445 * In 64-bit mode, mod == 0 and r/m == 5
446 * denotes %rip-relative addressing; in 32-bit
447 * mode, the base register isn't used. In both
448 * modes, there is a 32-bit operand.
449 */
450 if (mod == 0 && rm == 5) {
451 #if __sol64 || defined(__APPLE__)
452 if (p_model == DATAMODEL_LP64)
453 tp->ftt_base = REG_RIP;
454 else
455 #endif
456 tp->ftt_base = FASTTRAP_NOREG;
457 sz = 4;
458 } else {
459 uint8_t base = rm |
460 (FASTTRAP_REX_B(rex) << 3);
461
462 tp->ftt_base = regmap[base];
463 sz = mod == 1 ? 1 : mod == 2 ? 4 : 0;
464 }
465 tp->ftt_index = FASTTRAP_NOREG;
466 i = 2;
467 }
468
469 if (sz == 1)
470 tp->ftt_dest = *(int8_t *)&instr[start + i];
471 else if (sz == 4)
472 tp->ftt_dest = *(int32_t *)&instr[start + i];
473 else
474 tp->ftt_dest = 0;
475 }
476 } else {
477 switch (instr[start]) {
478 case FASTTRAP_RET:
479 tp->ftt_type = FASTTRAP_T_RET;
480 break;
481
482 case FASTTRAP_RET16:
483 tp->ftt_type = FASTTRAP_T_RET16;
484 tp->ftt_dest = *(uint16_t *)&instr[start + 1];
485 break;
486
487 case FASTTRAP_JO:
488 case FASTTRAP_JNO:
489 case FASTTRAP_JB:
490 case FASTTRAP_JAE:
491 case FASTTRAP_JE:
492 case FASTTRAP_JNE:
493 case FASTTRAP_JBE:
494 case FASTTRAP_JA:
495 case FASTTRAP_JS:
496 case FASTTRAP_JNS:
497 case FASTTRAP_JP:
498 case FASTTRAP_JNP:
499 case FASTTRAP_JL:
500 case FASTTRAP_JGE:
501 case FASTTRAP_JLE:
502 case FASTTRAP_JG:
503 tp->ftt_type = FASTTRAP_T_JCC;
504 tp->ftt_code = instr[start];
505 tp->ftt_dest = pc + tp->ftt_size +
506 (int8_t)instr[start + 1];
507 break;
508
509 case FASTTRAP_LOOPNZ:
510 case FASTTRAP_LOOPZ:
511 case FASTTRAP_LOOP:
512 tp->ftt_type = FASTTRAP_T_LOOP;
513 tp->ftt_code = instr[start];
514 tp->ftt_dest = pc + tp->ftt_size +
515 (int8_t)instr[start + 1];
516 break;
517
518 case FASTTRAP_JCXZ:
519 tp->ftt_type = FASTTRAP_T_JCXZ;
520 tp->ftt_dest = pc + tp->ftt_size +
521 (int8_t)instr[start + 1];
522 break;
523
524 case FASTTRAP_CALL:
525 tp->ftt_type = FASTTRAP_T_CALL;
526 tp->ftt_dest = pc + tp->ftt_size +
527 *(int32_t *)&instr[start + 1];
528 tp->ftt_code = 0;
529 break;
530
531 case FASTTRAP_JMP32:
532 tp->ftt_type = FASTTRAP_T_JMP;
533 tp->ftt_dest = pc + tp->ftt_size +
534 *(int32_t *)&instr[start + 1];
535 break;
536 case FASTTRAP_JMP8:
537 tp->ftt_type = FASTTRAP_T_JMP;
538 tp->ftt_dest = pc + tp->ftt_size +
539 (int8_t)instr[start + 1];
540 break;
541
542 case FASTTRAP_PUSHL_EBP:
543 if (start == 0)
544 tp->ftt_type = FASTTRAP_T_PUSHL_EBP;
545 break;
546
547 case FASTTRAP_NOP:
548 #if __sol64 || defined(__APPLE__)
549 ASSERT(p_model == DATAMODEL_LP64 || rex == 0);
550
551 /*
552 * On sol64 we have to be careful not to confuse a nop
553 * (actually xchgl %eax, %eax) with an instruction using
554 * the same opcode, but that does something different
555 * (e.g. xchgl %r8d, %eax or xcghq %r8, %rax).
556 */
557 if (FASTTRAP_REX_B(rex) == 0)
558 #endif
559 tp->ftt_type = FASTTRAP_T_NOP;
560 break;
561
562 case FASTTRAP_INT3:
563 /*
564 * The pid provider shares the int3 trap with debugger
565 * breakpoints so we can't instrument them.
566 */
567 ASSERT(instr[start] == FASTTRAP_INSTR);
568 return (-1);
569
570 case FASTTRAP_INT:
571 /*
572 * Interrupts seem like they could be traced with
573 * no negative implications, but it's possible that
574 * a thread could be redirected by the trap handling
575 * code which would eventually return to the
576 * instruction after the interrupt. If the interrupt
577 * were in our scratch space, the subsequent
578 * instruction might be overwritten before we return.
579 * Accordingly we refuse to instrument any interrupt.
580 */
581 return (-1);
582 }
583 }
584
585 #if __sol64 || defined(__APPLE__)
586 if (p_model == DATAMODEL_LP64 && tp->ftt_type == FASTTRAP_T_COMMON) {
587 /*
588 * If the process is 64-bit and the instruction type is still
589 * FASTTRAP_T_COMMON -- meaning we're going to copy it out an
590 * execute it -- we need to watch for %rip-relative
591 * addressing mode. See the portion of fasttrap_pid_probe()
592 * below where we handle tracepoints with type
593 * FASTTRAP_T_COMMON for how we emulate instructions that
594 * employ %rip-relative addressing.
595 */
596 if (rmindex != -1) {
597 uint_t mod = FASTTRAP_MODRM_MOD(instr[rmindex]);
598 uint_t reg = FASTTRAP_MODRM_REG(instr[rmindex]);
599 uint_t rm = FASTTRAP_MODRM_RM(instr[rmindex]);
600
601 ASSERT(rmindex > (int)start);
602
603 if (mod == 0 && rm == 5) {
604 /*
605 * We need to be sure to avoid other
606 * registers used by this instruction. While
607 * the reg field may determine the op code
608 * rather than denoting a register, assuming
609 * that it denotes a register is always safe.
610 * We leave the REX field intact and use
611 * whatever value's there for simplicity.
612 */
613 if (reg != 0) {
614 tp->ftt_ripmode = FASTTRAP_RIP_1 |
615 (FASTTRAP_RIP_X *
616 FASTTRAP_REX_B(rex));
617 rm = 0;
618 } else {
619 tp->ftt_ripmode = FASTTRAP_RIP_2 |
620 (FASTTRAP_RIP_X *
621 FASTTRAP_REX_B(rex));
622 rm = 1;
623 }
624
625 tp->ftt_modrm = tp->ftt_instr[rmindex];
626 tp->ftt_instr[rmindex] =
627 FASTTRAP_MODRM(2, reg, rm);
628 }
629 }
630 }
631 #endif
632
633 return (0);
634 }
635
636 int
637 fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp)
638 {
639 fasttrap_instr_t instr = FASTTRAP_INSTR;
640
641 if (uwrite(p, &instr, 1, tp->ftt_pc) != 0)
642 return (-1);
643
644 return (0);
645 }
646
647 int
648 fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp)
649 {
650 uint8_t instr;
651
652 /*
653 * Distinguish between read or write failures and a changed
654 * instruction.
655 */
656 if (uread(p, &instr, 1, tp->ftt_pc) != 0)
657 return (0);
658 if (instr != FASTTRAP_INSTR)
659 return (0);
660 if (uwrite(p, &tp->ftt_instr[0], 1, tp->ftt_pc) != 0)
661 return (-1);
662
663 return (0);
664 }
665
666 static void
667 fasttrap_return_common(x86_saved_state_t *regs, user_addr_t pc, pid_t pid,
668 user_addr_t new_pc)
669 {
670 x86_saved_state64_t *regs64;
671 x86_saved_state32_t *regs32;
672 unsigned int p_model;
673
674 if (is_saved_state64(regs)) {
675 regs64 = saved_state64(regs);
676 regs32 = NULL;
677 p_model = DATAMODEL_LP64;
678 } else {
679 regs64 = NULL;
680 regs32 = saved_state32(regs);
681 p_model = DATAMODEL_ILP32;
682 }
683
684 fasttrap_tracepoint_t *tp;
685 fasttrap_bucket_t *bucket;
686 fasttrap_id_t *id;
687 lck_mtx_t *pid_mtx;
688
689 pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
690 lck_mtx_lock(pid_mtx);
691 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
692
693 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
694 if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
695 !tp->ftt_proc->ftpc_defunct)
696 break;
697 }
698
699 /*
700 * Don't sweat it if we can't find the tracepoint again; unlike
701 * when we're in fasttrap_pid_probe(), finding the tracepoint here
702 * is not essential to the correct execution of the process.
703 */
704 if (tp == NULL) {
705 lck_mtx_unlock(pid_mtx);
706 return;
707 }
708
709 for (id = tp->ftt_retids; id != NULL; id = id->fti_next) {
710 /*
711 * If there's a branch that could act as a return site, we
712 * need to trace it, and check here if the program counter is
713 * external to the function.
714 */
715 if (tp->ftt_type != FASTTRAP_T_RET &&
716 tp->ftt_type != FASTTRAP_T_RET16 &&
717 new_pc - id->fti_probe->ftp_faddr <
718 id->fti_probe->ftp_fsize)
719 continue;
720
721 if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) {
722 dtrace_probe(dtrace_probeid_error, 0 /* state */, id->fti_probe->ftp_id,
723 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV);
724 } else if (p_model == DATAMODEL_LP64) {
725 dtrace_probe(id->fti_probe->ftp_id,
726 pc - id->fti_probe->ftp_faddr,
727 regs64->rax, regs64->rdx, 0, 0);
728 } else {
729 dtrace_probe(id->fti_probe->ftp_id,
730 pc - id->fti_probe->ftp_faddr,
731 regs32->eax, regs32->edx, 0, 0);
732 }
733 }
734
735 lck_mtx_unlock(pid_mtx);
736 }
737
738 static void
739 fasttrap_sigsegv(proc_t *p, uthread_t t, user_addr_t addr)
740 {
741 proc_lock(p);
742
743 /* Set fault address and mark signal */
744 t->uu_code = addr;
745 t->uu_siglist |= sigmask(SIGSEGV);
746
747 /*
748 * XXX These two line may be redundant; if not, then we need
749 * XXX to potentially set the data address in the machine
750 * XXX specific thread state structure to indicate the address.
751 */
752 t->uu_exception = KERN_INVALID_ADDRESS; /* SIGSEGV */
753 t->uu_subcode = 0; /* XXX pad */
754
755 proc_unlock(p);
756
757 /* raise signal */
758 signal_setast(t->uu_context.vc_thread);
759 }
760
761 static void
762 fasttrap_usdt_args64(fasttrap_probe_t *probe, x86_saved_state64_t *regs64, int argc,
763 uint64_t *argv)
764 {
765 int i, x, cap = MIN(argc, probe->ftp_nargs);
766 user_addr_t stack = (user_addr_t)regs64->isf.rsp;
767
768 for (i = 0; i < cap; i++) {
769 x = probe->ftp_argmap[i];
770
771 if (x < 6) {
772 /* FIXME! This may be broken, needs testing */
773 argv[i] = (&regs64->rdi)[x];
774 } else {
775 fasttrap_fuword64_noerr(stack + (x * sizeof(uint64_t)), &argv[i]);
776 }
777 }
778
779 for (; i < argc; i++) {
780 argv[i] = 0;
781 }
782 }
783
784 static void
785 fasttrap_usdt_args32(fasttrap_probe_t *probe, x86_saved_state32_t *regs32, int argc,
786 uint32_t *argv)
787 {
788 int i, x, cap = MIN(argc, probe->ftp_nargs);
789 uint32_t *stack = (uint32_t *)regs32->uesp;
790
791 for (i = 0; i < cap; i++) {
792 x = probe->ftp_argmap[i];
793
794 fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[x], &argv[i]);
795 }
796
797 for (; i < argc; i++) {
798 argv[i] = 0;
799 }
800 }
801
802 /*
803 * FIXME!
804 */
805 static int
806 fasttrap_do_seg(fasttrap_tracepoint_t *tp, x86_saved_state_t *rp, user_addr_t *addr) // 64 bit
807 {
808 #pragma unused(tp, rp, addr)
809 printf("fasttrap_do_seg() called while unimplemented.\n");
810 #if 0
811 proc_t *p = curproc;
812 user_desc_t *desc;
813 uint16_t sel, ndx, type;
814 uintptr_t limit;
815
816 switch (tp->ftt_segment) {
817 case FASTTRAP_SEG_CS:
818 sel = rp->r_cs;
819 break;
820 case FASTTRAP_SEG_DS:
821 sel = rp->r_ds;
822 break;
823 case FASTTRAP_SEG_ES:
824 sel = rp->r_es;
825 break;
826 case FASTTRAP_SEG_FS:
827 sel = rp->r_fs;
828 break;
829 case FASTTRAP_SEG_GS:
830 sel = rp->r_gs;
831 break;
832 case FASTTRAP_SEG_SS:
833 sel = rp->r_ss;
834 break;
835 }
836
837 /*
838 * Make sure the given segment register specifies a user priority
839 * selector rather than a kernel selector.
840 */
841 if (!SELISUPL(sel))
842 return (-1);
843
844 ndx = SELTOIDX(sel);
845
846 /*
847 * Check the bounds and grab the descriptor out of the specified
848 * descriptor table.
849 */
850 if (SELISLDT(sel)) {
851 if (ndx > p->p_ldtlimit)
852 return (-1);
853
854 desc = p->p_ldt + ndx;
855
856 } else {
857 if (ndx >= NGDT)
858 return (-1);
859
860 desc = cpu_get_gdt() + ndx;
861 }
862
863 /*
864 * The descriptor must have user privilege level and it must be
865 * present in memory.
866 */
867 if (desc->usd_dpl != SEL_UPL || desc->usd_p != 1)
868 return (-1);
869
870 type = desc->usd_type;
871
872 /*
873 * If the S bit in the type field is not set, this descriptor can
874 * only be used in system context.
875 */
876 if ((type & 0x10) != 0x10)
877 return (-1);
878
879 limit = USEGD_GETLIMIT(desc) * (desc->usd_gran ? PAGESIZE : 1);
880
881 if (tp->ftt_segment == FASTTRAP_SEG_CS) {
882 /*
883 * The code/data bit and readable bit must both be set.
884 */
885 if ((type & 0xa) != 0xa)
886 return (-1);
887
888 if (*addr > limit)
889 return (-1);
890 } else {
891 /*
892 * The code/data bit must be clear.
893 */
894 if ((type & 0x8) != 0)
895 return (-1);
896
897 /*
898 * If the expand-down bit is clear, we just check the limit as
899 * it would naturally be applied. Otherwise, we need to check
900 * that the address is the range [limit + 1 .. 0xffff] or
901 * [limit + 1 ... 0xffffffff] depending on if the default
902 * operand size bit is set.
903 */
904 if ((type & 0x4) == 0) {
905 if (*addr > limit)
906 return (-1);
907 } else if (desc->usd_def32) {
908 if (*addr < limit + 1 || 0xffff < *addr)
909 return (-1);
910 } else {
911 if (*addr < limit + 1 || 0xffffffff < *addr)
912 return (-1);
913 }
914 }
915
916 *addr += USEGD_GETBASE(desc);
917 #endif /* 0 */
918 return (0);
919 }
920
921 /*
922 * Due to variances between Solaris and xnu, I have split this into a 32 bit and 64 bit
923 * code path. It still takes an x86_saved_state_t* argument, because it must sometimes
924 * call other methods that require a x86_saved_state_t.
925 *
926 * NOTE!!!!
927 *
928 * Any changes made to this method must be echo'd in fasttrap_pid_probe64!
929 *
930 */
931 static int
932 fasttrap_pid_probe32(x86_saved_state_t *regs)
933 {
934 ASSERT(is_saved_state32(regs));
935
936 x86_saved_state32_t *regs32 = saved_state32(regs);
937 user_addr_t pc = regs32->eip - 1;
938 proc_t *p = current_proc();
939 user_addr_t new_pc = 0;
940 fasttrap_bucket_t *bucket;
941 lck_mtx_t *pid_mtx;
942 fasttrap_tracepoint_t *tp, tp_local;
943 pid_t pid;
944 dtrace_icookie_t cookie;
945 uint_t is_enabled = 0;
946
947 uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
948
949 /*
950 * It's possible that a user (in a veritable orgy of bad planning)
951 * could redirect this thread's flow of control before it reached the
952 * return probe fasttrap. In this case we need to kill the process
953 * since it's in a unrecoverable state.
954 */
955 if (uthread->t_dtrace_step) {
956 ASSERT(uthread->t_dtrace_on);
957 fasttrap_sigtrap(p, uthread, pc);
958 return (0);
959 }
960
961 /*
962 * Clear all user tracing flags.
963 */
964 uthread->t_dtrace_ft = 0;
965 uthread->t_dtrace_pc = 0;
966 uthread->t_dtrace_npc = 0;
967 uthread->t_dtrace_scrpc = 0;
968 uthread->t_dtrace_astpc = 0;
969
970 /*
971 * Treat a child created by a call to vfork(2) as if it were its
972 * parent. We know that there's only one thread of control in such a
973 * process: this one.
974 */
975 /*
976 * APPLE NOTE: Terry says: "You need to hold the process locks (currently: kernel funnel) for this traversal"
977 * FIXME: How do we assert this?
978 */
979 while (p->p_lflag & P_LINVFORK)
980 p = p->p_pptr;
981
982 pid = p->p_pid;
983 pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
984 lck_mtx_lock(pid_mtx);
985 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
986
987 /*
988 * Lookup the tracepoint that the process just hit.
989 */
990 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
991 if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
992 !tp->ftt_proc->ftpc_defunct)
993 break;
994 }
995
996 /*
997 * If we couldn't find a matching tracepoint, either a tracepoint has
998 * been inserted without using the pid<pid> ioctl interface (see
999 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
1000 */
1001 if (tp == NULL) {
1002 lck_mtx_unlock(pid_mtx);
1003 return (-1);
1004 }
1005
1006 /*
1007 * Set the program counter to the address of the traced instruction
1008 * so that it looks right in ustack() output.
1009 */
1010 regs32->eip = pc;
1011
1012 if (tp->ftt_ids != NULL) {
1013 fasttrap_id_t *id;
1014
1015 uint32_t s0, s1, s2, s3, s4, s5;
1016 uint32_t *stack = (uint32_t *)regs32->uesp;
1017
1018 /*
1019 * In 32-bit mode, all arguments are passed on the
1020 * stack. If this is a function entry probe, we need
1021 * to skip the first entry on the stack as it
1022 * represents the return address rather than a
1023 * parameter to the function.
1024 */
1025 fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[0], &s0);
1026 fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[1], &s1);
1027 fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[2], &s2);
1028 fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[3], &s3);
1029 fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[4], &s4);
1030 fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[5], &s5);
1031
1032 for (id = tp->ftt_ids; id != NULL; id = id->fti_next) {
1033 fasttrap_probe_t *probe = id->fti_probe;
1034
1035 if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) {
1036 dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id,
1037 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV);
1038 } else if (id->fti_ptype == DTFTP_ENTRY) {
1039 /*
1040 * We note that this was an entry
1041 * probe to help ustack() find the
1042 * first caller.
1043 */
1044 cookie = dtrace_interrupt_disable();
1045 DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY);
1046 dtrace_probe(probe->ftp_id, s1, s2,
1047 s3, s4, s5);
1048 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY);
1049 dtrace_interrupt_enable(cookie);
1050 } else if (id->fti_ptype == DTFTP_IS_ENABLED) {
1051 /*
1052 * Note that in this case, we don't
1053 * call dtrace_probe() since it's only
1054 * an artificial probe meant to change
1055 * the flow of control so that it
1056 * encounters the true probe.
1057 */
1058 is_enabled = 1;
1059 } else if (probe->ftp_argmap == NULL) {
1060 dtrace_probe(probe->ftp_id, s0, s1,
1061 s2, s3, s4);
1062 } else {
1063 uint32_t t[5];
1064
1065 fasttrap_usdt_args32(probe, regs32,
1066 sizeof (t) / sizeof (t[0]), t);
1067
1068 dtrace_probe(probe->ftp_id, t[0], t[1],
1069 t[2], t[3], t[4]);
1070 }
1071
1072 /* APPLE NOTE: Oneshot probes get one and only one chance... */
1073 if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) {
1074 fasttrap_tracepoint_remove(p, tp);
1075 }
1076 }
1077 }
1078
1079 /*
1080 * We're about to do a bunch of work so we cache a local copy of
1081 * the tracepoint to emulate the instruction, and then find the
1082 * tracepoint again later if we need to light up any return probes.
1083 */
1084 tp_local = *tp;
1085 lck_mtx_unlock(pid_mtx);
1086 tp = &tp_local;
1087
1088 /*
1089 * Set the program counter to appear as though the traced instruction
1090 * had completely executed. This ensures that fasttrap_getreg() will
1091 * report the expected value for REG_RIP.
1092 */
1093 regs32->eip = pc + tp->ftt_size;
1094
1095 /*
1096 * If there's an is-enabled probe connected to this tracepoint it
1097 * means that there was a 'xorl %eax, %eax' or 'xorq %rax, %rax'
1098 * instruction that was placed there by DTrace when the binary was
1099 * linked. As this probe is, in fact, enabled, we need to stuff 1
1100 * into %eax or %rax. Accordingly, we can bypass all the instruction
1101 * emulation logic since we know the inevitable result. It's possible
1102 * that a user could construct a scenario where the 'is-enabled'
1103 * probe was on some other instruction, but that would be a rather
1104 * exotic way to shoot oneself in the foot.
1105 */
1106 if (is_enabled) {
1107 regs32->eax = 1;
1108 new_pc = regs32->eip;
1109 goto done;
1110 }
1111
1112 /*
1113 * We emulate certain types of instructions to ensure correctness
1114 * (in the case of position dependent instructions) or optimize
1115 * common cases. The rest we have the thread execute back in user-
1116 * land.
1117 */
1118 switch (tp->ftt_type) {
1119 case FASTTRAP_T_RET:
1120 case FASTTRAP_T_RET16:
1121 {
1122 user_addr_t dst;
1123 user_addr_t addr;
1124 int ret;
1125
1126 /*
1127 * We have to emulate _every_ facet of the behavior of a ret
1128 * instruction including what happens if the load from %esp
1129 * fails; in that case, we send a SIGSEGV.
1130 */
1131 uint32_t dst32;
1132 ret = fasttrap_fuword32((user_addr_t)regs32->uesp, &dst32);
1133 dst = dst32;
1134 addr = regs32->uesp + sizeof (uint32_t);
1135
1136 if (ret == -1) {
1137 fasttrap_sigsegv(p, uthread, (user_addr_t)regs32->uesp);
1138 new_pc = pc;
1139 break;
1140 }
1141
1142 if (tp->ftt_type == FASTTRAP_T_RET16)
1143 addr += tp->ftt_dest;
1144
1145 regs32->uesp = addr;
1146 new_pc = dst;
1147 break;
1148 }
1149
1150 case FASTTRAP_T_JCC:
1151 {
1152 uint_t taken;
1153
1154 switch (tp->ftt_code) {
1155 case FASTTRAP_JO:
1156 taken = (regs32->efl & FASTTRAP_EFLAGS_OF) != 0;
1157 break;
1158 case FASTTRAP_JNO:
1159 taken = (regs32->efl & FASTTRAP_EFLAGS_OF) == 0;
1160 break;
1161 case FASTTRAP_JB:
1162 taken = (regs32->efl & FASTTRAP_EFLAGS_CF) != 0;
1163 break;
1164 case FASTTRAP_JAE:
1165 taken = (regs32->efl & FASTTRAP_EFLAGS_CF) == 0;
1166 break;
1167 case FASTTRAP_JE:
1168 taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) != 0;
1169 break;
1170 case FASTTRAP_JNE:
1171 taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) == 0;
1172 break;
1173 case FASTTRAP_JBE:
1174 taken = (regs32->efl & FASTTRAP_EFLAGS_CF) != 0 ||
1175 (regs32->efl & FASTTRAP_EFLAGS_ZF) != 0;
1176 break;
1177 case FASTTRAP_JA:
1178 taken = (regs32->efl & FASTTRAP_EFLAGS_CF) == 0 &&
1179 (regs32->efl & FASTTRAP_EFLAGS_ZF) == 0;
1180 break;
1181 case FASTTRAP_JS:
1182 taken = (regs32->efl & FASTTRAP_EFLAGS_SF) != 0;
1183 break;
1184 case FASTTRAP_JNS:
1185 taken = (regs32->efl & FASTTRAP_EFLAGS_SF) == 0;
1186 break;
1187 case FASTTRAP_JP:
1188 taken = (regs32->efl & FASTTRAP_EFLAGS_PF) != 0;
1189 break;
1190 case FASTTRAP_JNP:
1191 taken = (regs32->efl & FASTTRAP_EFLAGS_PF) == 0;
1192 break;
1193 case FASTTRAP_JL:
1194 taken = ((regs32->efl & FASTTRAP_EFLAGS_SF) == 0) !=
1195 ((regs32->efl & FASTTRAP_EFLAGS_OF) == 0);
1196 break;
1197 case FASTTRAP_JGE:
1198 taken = ((regs32->efl & FASTTRAP_EFLAGS_SF) == 0) ==
1199 ((regs32->efl & FASTTRAP_EFLAGS_OF) == 0);
1200 break;
1201 case FASTTRAP_JLE:
1202 taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) != 0 ||
1203 ((regs32->efl & FASTTRAP_EFLAGS_SF) == 0) !=
1204 ((regs32->efl & FASTTRAP_EFLAGS_OF) == 0);
1205 break;
1206 case FASTTRAP_JG:
1207 taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) == 0 &&
1208 ((regs32->efl & FASTTRAP_EFLAGS_SF) == 0) ==
1209 ((regs32->efl & FASTTRAP_EFLAGS_OF) == 0);
1210 break;
1211 default:
1212 taken = FALSE;
1213 }
1214
1215 if (taken)
1216 new_pc = tp->ftt_dest;
1217 else
1218 new_pc = pc + tp->ftt_size;
1219 break;
1220 }
1221
1222 case FASTTRAP_T_LOOP:
1223 {
1224 uint_t taken;
1225 greg_t cx = regs32->ecx--;
1226
1227 switch (tp->ftt_code) {
1228 case FASTTRAP_LOOPNZ:
1229 taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) == 0 &&
1230 cx != 0;
1231 break;
1232 case FASTTRAP_LOOPZ:
1233 taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) != 0 &&
1234 cx != 0;
1235 break;
1236 case FASTTRAP_LOOP:
1237 taken = (cx != 0);
1238 break;
1239 default:
1240 taken = FALSE;
1241 }
1242
1243 if (taken)
1244 new_pc = tp->ftt_dest;
1245 else
1246 new_pc = pc + tp->ftt_size;
1247 break;
1248 }
1249
1250 case FASTTRAP_T_JCXZ:
1251 {
1252 greg_t cx = regs32->ecx;
1253
1254 if (cx == 0)
1255 new_pc = tp->ftt_dest;
1256 else
1257 new_pc = pc + tp->ftt_size;
1258 break;
1259 }
1260
1261 case FASTTRAP_T_PUSHL_EBP:
1262 {
1263 user_addr_t addr = regs32->uesp - sizeof (uint32_t);
1264 int ret = fasttrap_suword32(addr, (uint32_t)regs32->ebp);
1265
1266 if (ret == -1) {
1267 fasttrap_sigsegv(p, uthread, addr);
1268 new_pc = pc;
1269 break;
1270 }
1271
1272 regs32->uesp = addr;
1273 new_pc = pc + tp->ftt_size;
1274 break;
1275 }
1276
1277 case FASTTRAP_T_NOP:
1278 new_pc = pc + tp->ftt_size;
1279 break;
1280
1281 case FASTTRAP_T_JMP:
1282 case FASTTRAP_T_CALL:
1283 if (tp->ftt_code == 0) {
1284 new_pc = tp->ftt_dest;
1285 } else {
1286 user_addr_t /* value ,*/ addr = tp->ftt_dest;
1287
1288 if (tp->ftt_base != FASTTRAP_NOREG)
1289 addr += fasttrap_getreg(regs, tp->ftt_base);
1290 if (tp->ftt_index != FASTTRAP_NOREG)
1291 addr += fasttrap_getreg(regs, tp->ftt_index) <<
1292 tp->ftt_scale;
1293
1294 if (tp->ftt_code == 1) {
1295 /*
1296 * If there's a segment prefix for this
1297 * instruction, we'll need to check permissions
1298 * and bounds on the given selector, and adjust
1299 * the address accordingly.
1300 */
1301 if (tp->ftt_segment != FASTTRAP_SEG_NONE &&
1302 fasttrap_do_seg(tp, regs, &addr) != 0) {
1303 fasttrap_sigsegv(p, uthread, addr);
1304 new_pc = pc;
1305 break;
1306 }
1307
1308 uint32_t value32;
1309 addr = (user_addr_t)(uint32_t)addr;
1310 if (fasttrap_fuword32(addr, &value32) == -1) {
1311 fasttrap_sigsegv(p, uthread, addr);
1312 new_pc = pc;
1313 break;
1314 }
1315 new_pc = value32;
1316 } else {
1317 new_pc = addr;
1318 }
1319 }
1320
1321 /*
1322 * If this is a call instruction, we need to push the return
1323 * address onto the stack. If this fails, we send the process
1324 * a SIGSEGV and reset the pc to emulate what would happen if
1325 * this instruction weren't traced.
1326 */
1327 if (tp->ftt_type == FASTTRAP_T_CALL) {
1328 user_addr_t addr = regs32->uesp - sizeof (uint32_t);
1329 int ret = fasttrap_suword32(addr, (uint32_t)(pc + tp->ftt_size));
1330
1331 if (ret == -1) {
1332 fasttrap_sigsegv(p, uthread, addr);
1333 new_pc = pc;
1334 break;
1335 }
1336
1337 regs32->uesp = addr;
1338 }
1339 break;
1340
1341 case FASTTRAP_T_COMMON:
1342 {
1343 user_addr_t addr;
1344 uint8_t scratch[2 * FASTTRAP_MAX_INSTR_SIZE + 5 + 2];
1345 uint_t i = 0;
1346
1347 /*
1348 * Generic Instruction Tracing
1349 * ---------------------------
1350 *
1351 * This is the layout of the scratch space in the user-land
1352 * thread structure for our generated instructions.
1353 *
1354 * 32-bit mode bytes
1355 * ------------------------ -----
1356 * a: <original instruction> <= 15
1357 * jmp <pc + tp->ftt_size> 5
1358 * b: <original instrction> <= 15
1359 * int T_DTRACE_RET 2
1360 * -----
1361 * <= 37
1362 *
1363 * 64-bit mode bytes
1364 * ------------------------ -----
1365 * a: <original instruction> <= 15
1366 * jmp 0(%rip) 6
1367 * <pc + tp->ftt_size> 8
1368 * b: <original instruction> <= 15
1369 * int T_DTRACE_RET 2
1370 * -----
1371 * <= 46
1372 *
1373 * The %pc is set to a, and curthread->t_dtrace_astpc is set
1374 * to b. If we encounter a signal on the way out of the
1375 * kernel, trap() will set %pc to curthread->t_dtrace_astpc
1376 * so that we execute the original instruction and re-enter
1377 * the kernel rather than redirecting to the next instruction.
1378 *
1379 * If there are return probes (so we know that we're going to
1380 * need to reenter the kernel after executing the original
1381 * instruction), the scratch space will just contain the
1382 * original instruction followed by an interrupt -- the same
1383 * data as at b.
1384 */
1385
1386 addr = uthread->t_dtrace_scratch->addr;
1387
1388 if (addr == 0LL) {
1389 fasttrap_sigtrap(p, uthread, pc); // Should be killing target proc
1390 new_pc = pc;
1391 break;
1392 }
1393
1394 ASSERT(tp->ftt_size < FASTTRAP_MAX_INSTR_SIZE);
1395
1396 uthread->t_dtrace_scrpc = addr;
1397 bcopy(tp->ftt_instr, &scratch[i], tp->ftt_size);
1398 i += tp->ftt_size;
1399
1400 /*
1401 * Set up the jmp to the next instruction; note that
1402 * the size of the traced instruction cancels out.
1403 */
1404 scratch[i++] = FASTTRAP_JMP32;
1405 *(uint32_t *)&scratch[i] = pc - addr - 5;
1406 i += sizeof (uint32_t);
1407
1408 uthread->t_dtrace_astpc = addr + i;
1409 bcopy(tp->ftt_instr, &scratch[i], tp->ftt_size);
1410 i += tp->ftt_size;
1411 scratch[i++] = FASTTRAP_INT;
1412 scratch[i++] = T_DTRACE_RET;
1413
1414 if (fasttrap_copyout(scratch, addr, i)) {
1415 fasttrap_sigtrap(p, uthread, pc);
1416 new_pc = pc;
1417 break;
1418 }
1419
1420 if (tp->ftt_retids != NULL) {
1421 uthread->t_dtrace_step = 1;
1422 uthread->t_dtrace_ret = 1;
1423 new_pc = uthread->t_dtrace_astpc;
1424 } else {
1425 new_pc = uthread->t_dtrace_scrpc;
1426 }
1427
1428 uthread->t_dtrace_pc = pc;
1429 uthread->t_dtrace_npc = pc + tp->ftt_size;
1430 uthread->t_dtrace_on = 1;
1431 break;
1432 }
1433
1434 default:
1435 panic("fasttrap: mishandled an instruction");
1436 }
1437
1438 done:
1439 /*
1440 * APPLE NOTE:
1441 *
1442 * We're setting this earlier than Solaris does, to get a "correct"
1443 * ustack() output. In the Sun code, a() -> b() -> c() -> d() is
1444 * reported at: d, b, a. The new way gives c, b, a, which is closer
1445 * to correct, as the return instruction has already exectued.
1446 */
1447 regs32->eip = new_pc;
1448
1449 /*
1450 * If there were no return probes when we first found the tracepoint,
1451 * we should feel no obligation to honor any return probes that were
1452 * subsequently enabled -- they'll just have to wait until the next
1453 * time around.
1454 */
1455 if (tp->ftt_retids != NULL) {
1456 /*
1457 * We need to wait until the results of the instruction are
1458 * apparent before invoking any return probes. If this
1459 * instruction was emulated we can just call
1460 * fasttrap_return_common(); if it needs to be executed, we
1461 * need to wait until the user thread returns to the kernel.
1462 */
1463 if (tp->ftt_type != FASTTRAP_T_COMMON) {
1464 fasttrap_return_common(regs, pc, pid, new_pc);
1465 } else {
1466 ASSERT(uthread->t_dtrace_ret != 0);
1467 ASSERT(uthread->t_dtrace_pc == pc);
1468 ASSERT(uthread->t_dtrace_scrpc != 0);
1469 ASSERT(new_pc == uthread->t_dtrace_astpc);
1470 }
1471 }
1472
1473 return (0);
1474 }
1475
1476 /*
1477 * Due to variances between Solaris and xnu, I have split this into a 32 bit and 64 bit
1478 * code path. It still takes an x86_saved_state_t* argument, because it must sometimes
1479 * call other methods that require a x86_saved_state_t.
1480 *
1481 * NOTE!!!!
1482 *
1483 * Any changes made to this method must be echo'd in fasttrap_pid_probe32!
1484 *
1485 */
1486 static int
1487 fasttrap_pid_probe64(x86_saved_state_t *regs)
1488 {
1489 ASSERT(is_saved_state64(regs));
1490
1491 x86_saved_state64_t *regs64 = saved_state64(regs);
1492 user_addr_t pc = regs64->isf.rip - 1;
1493 proc_t *p = current_proc();
1494 user_addr_t new_pc = 0;
1495 fasttrap_bucket_t *bucket;
1496 lck_mtx_t *pid_mtx;
1497 fasttrap_tracepoint_t *tp, tp_local;
1498 pid_t pid;
1499 dtrace_icookie_t cookie;
1500 uint_t is_enabled = 0;
1501
1502 uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
1503
1504 /*
1505 * It's possible that a user (in a veritable orgy of bad planning)
1506 * could redirect this thread's flow of control before it reached the
1507 * return probe fasttrap. In this case we need to kill the process
1508 * since it's in a unrecoverable state.
1509 */
1510 if (uthread->t_dtrace_step) {
1511 ASSERT(uthread->t_dtrace_on);
1512 fasttrap_sigtrap(p, uthread, pc);
1513 return (0);
1514 }
1515
1516 /*
1517 * Clear all user tracing flags.
1518 */
1519 uthread->t_dtrace_ft = 0;
1520 uthread->t_dtrace_pc = 0;
1521 uthread->t_dtrace_npc = 0;
1522 uthread->t_dtrace_scrpc = 0;
1523 uthread->t_dtrace_astpc = 0;
1524 uthread->t_dtrace_regv = 0;
1525
1526 /*
1527 * Treat a child created by a call to vfork(2) as if it were its
1528 * parent. We know that there's only one thread of control in such a
1529 * process: this one.
1530 */
1531 /*
1532 * APPLE NOTE: Terry says: "You need to hold the process locks (currently: kernel funnel) for this traversal"
1533 * FIXME: How do we assert this?
1534 */
1535 while (p->p_lflag & P_LINVFORK)
1536 p = p->p_pptr;
1537
1538 pid = p->p_pid;
1539 pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
1540 lck_mtx_lock(pid_mtx);
1541 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
1542
1543 /*
1544 * Lookup the tracepoint that the process just hit.
1545 */
1546 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
1547 if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
1548 !tp->ftt_proc->ftpc_defunct)
1549 break;
1550 }
1551
1552 /*
1553 * If we couldn't find a matching tracepoint, either a tracepoint has
1554 * been inserted without using the pid<pid> ioctl interface (see
1555 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
1556 */
1557 if (tp == NULL) {
1558 lck_mtx_unlock(pid_mtx);
1559 return (-1);
1560 }
1561
1562 /*
1563 * Set the program counter to the address of the traced instruction
1564 * so that it looks right in ustack() output.
1565 */
1566 regs64->isf.rip = pc;
1567
1568 if (tp->ftt_ids != NULL) {
1569 fasttrap_id_t *id;
1570
1571 for (id = tp->ftt_ids; id != NULL; id = id->fti_next) {
1572 fasttrap_probe_t *probe = id->fti_probe;
1573
1574 if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) {
1575 dtrace_probe(dtrace_probeid_error, 0 /* state */, probe->ftp_id,
1576 1 /* ndx */, -1 /* offset */, DTRACEFLT_UPRIV);
1577 } else if (id->fti_ptype == DTFTP_ENTRY) {
1578 /*
1579 * We note that this was an entry
1580 * probe to help ustack() find the
1581 * first caller.
1582 */
1583 cookie = dtrace_interrupt_disable();
1584 DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY);
1585 dtrace_probe(probe->ftp_id, regs64->rdi,
1586 regs64->rsi, regs64->rdx, regs64->rcx,
1587 regs64->r8);
1588 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY);
1589 dtrace_interrupt_enable(cookie);
1590 } else if (id->fti_ptype == DTFTP_IS_ENABLED) {
1591 /*
1592 * Note that in this case, we don't
1593 * call dtrace_probe() since it's only
1594 * an artificial probe meant to change
1595 * the flow of control so that it
1596 * encounters the true probe.
1597 */
1598 is_enabled = 1;
1599 } else if (probe->ftp_argmap == NULL) {
1600 dtrace_probe(probe->ftp_id, regs64->rdi,
1601 regs64->rsi, regs64->rdx, regs64->rcx,
1602 regs64->r8);
1603 } else {
1604 uint64_t t[5];
1605
1606 fasttrap_usdt_args64(probe, regs64,
1607 sizeof (t) / sizeof (t[0]), t);
1608
1609 dtrace_probe(probe->ftp_id, t[0], t[1],
1610 t[2], t[3], t[4]);
1611 }
1612
1613 /* APPLE NOTE: Oneshot probes get one and only one chance... */
1614 if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) {
1615 fasttrap_tracepoint_remove(p, tp);
1616 }
1617 }
1618 }
1619
1620 /*
1621 * We're about to do a bunch of work so we cache a local copy of
1622 * the tracepoint to emulate the instruction, and then find the
1623 * tracepoint again later if we need to light up any return probes.
1624 */
1625 tp_local = *tp;
1626 lck_mtx_unlock(pid_mtx);
1627 tp = &tp_local;
1628
1629 /*
1630 * Set the program counter to appear as though the traced instruction
1631 * had completely executed. This ensures that fasttrap_getreg() will
1632 * report the expected value for REG_RIP.
1633 */
1634 regs64->isf.rip = pc + tp->ftt_size;
1635
1636 /*
1637 * If there's an is-enabled probe connected to this tracepoint it
1638 * means that there was a 'xorl %eax, %eax' or 'xorq %rax, %rax'
1639 * instruction that was placed there by DTrace when the binary was
1640 * linked. As this probe is, in fact, enabled, we need to stuff 1
1641 * into %eax or %rax. Accordingly, we can bypass all the instruction
1642 * emulation logic since we know the inevitable result. It's possible
1643 * that a user could construct a scenario where the 'is-enabled'
1644 * probe was on some other instruction, but that would be a rather
1645 * exotic way to shoot oneself in the foot.
1646 */
1647 if (is_enabled) {
1648 regs64->rax = 1;
1649 new_pc = regs64->isf.rip;
1650 goto done;
1651 }
1652
1653 /*
1654 * We emulate certain types of instructions to ensure correctness
1655 * (in the case of position dependent instructions) or optimize
1656 * common cases. The rest we have the thread execute back in user-
1657 * land.
1658 */
1659 switch (tp->ftt_type) {
1660 case FASTTRAP_T_RET:
1661 case FASTTRAP_T_RET16:
1662 {
1663 user_addr_t dst;
1664 user_addr_t addr;
1665 int ret;
1666
1667 /*
1668 * We have to emulate _every_ facet of the behavior of a ret
1669 * instruction including what happens if the load from %esp
1670 * fails; in that case, we send a SIGSEGV.
1671 */
1672 ret = fasttrap_fuword64((user_addr_t)regs64->isf.rsp, &dst);
1673 addr = regs64->isf.rsp + sizeof (uint64_t);
1674
1675 if (ret == -1) {
1676 fasttrap_sigsegv(p, uthread, (user_addr_t)regs64->isf.rsp);
1677 new_pc = pc;
1678 break;
1679 }
1680
1681 if (tp->ftt_type == FASTTRAP_T_RET16)
1682 addr += tp->ftt_dest;
1683
1684 regs64->isf.rsp = addr;
1685 new_pc = dst;
1686 break;
1687 }
1688
1689 case FASTTRAP_T_JCC:
1690 {
1691 uint_t taken;
1692
1693 switch (tp->ftt_code) {
1694 case FASTTRAP_JO:
1695 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_OF) != 0;
1696 break;
1697 case FASTTRAP_JNO:
1698 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_OF) == 0;
1699 break;
1700 case FASTTRAP_JB:
1701 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_CF) != 0;
1702 break;
1703 case FASTTRAP_JAE:
1704 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_CF) == 0;
1705 break;
1706 case FASTTRAP_JE:
1707 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) != 0;
1708 break;
1709 case FASTTRAP_JNE:
1710 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) == 0;
1711 break;
1712 case FASTTRAP_JBE:
1713 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_CF) != 0 ||
1714 (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) != 0;
1715 break;
1716 case FASTTRAP_JA:
1717 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_CF) == 0 &&
1718 (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) == 0;
1719 break;
1720 case FASTTRAP_JS:
1721 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_SF) != 0;
1722 break;
1723 case FASTTRAP_JNS:
1724 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_SF) == 0;
1725 break;
1726 case FASTTRAP_JP:
1727 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_PF) != 0;
1728 break;
1729 case FASTTRAP_JNP:
1730 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_PF) == 0;
1731 break;
1732 case FASTTRAP_JL:
1733 taken = ((regs64->isf.rflags & FASTTRAP_EFLAGS_SF) == 0) !=
1734 ((regs64->isf.rflags & FASTTRAP_EFLAGS_OF) == 0);
1735 break;
1736 case FASTTRAP_JGE:
1737 taken = ((regs64->isf.rflags & FASTTRAP_EFLAGS_SF) == 0) ==
1738 ((regs64->isf.rflags & FASTTRAP_EFLAGS_OF) == 0);
1739 break;
1740 case FASTTRAP_JLE:
1741 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) != 0 ||
1742 ((regs64->isf.rflags & FASTTRAP_EFLAGS_SF) == 0) !=
1743 ((regs64->isf.rflags & FASTTRAP_EFLAGS_OF) == 0);
1744 break;
1745 case FASTTRAP_JG:
1746 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) == 0 &&
1747 ((regs64->isf.rflags & FASTTRAP_EFLAGS_SF) == 0) ==
1748 ((regs64->isf.rflags & FASTTRAP_EFLAGS_OF) == 0);
1749 break;
1750 default:
1751 taken = FALSE;
1752 }
1753
1754 if (taken)
1755 new_pc = tp->ftt_dest;
1756 else
1757 new_pc = pc + tp->ftt_size;
1758 break;
1759 }
1760
1761 case FASTTRAP_T_LOOP:
1762 {
1763 uint_t taken;
1764 uint64_t cx = regs64->rcx--;
1765
1766 switch (tp->ftt_code) {
1767 case FASTTRAP_LOOPNZ:
1768 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) == 0 &&
1769 cx != 0;
1770 break;
1771 case FASTTRAP_LOOPZ:
1772 taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) != 0 &&
1773 cx != 0;
1774 break;
1775 case FASTTRAP_LOOP:
1776 taken = (cx != 0);
1777 break;
1778 default:
1779 taken = FALSE;
1780 }
1781
1782 if (taken)
1783 new_pc = tp->ftt_dest;
1784 else
1785 new_pc = pc + tp->ftt_size;
1786 break;
1787 }
1788
1789 case FASTTRAP_T_JCXZ:
1790 {
1791 uint64_t cx = regs64->rcx;
1792
1793 if (cx == 0)
1794 new_pc = tp->ftt_dest;
1795 else
1796 new_pc = pc + tp->ftt_size;
1797 break;
1798 }
1799
1800 case FASTTRAP_T_PUSHL_EBP:
1801 {
1802 user_addr_t addr = regs64->isf.rsp - sizeof (uint64_t);
1803 int ret = fasttrap_suword64(addr, (uint64_t)regs64->rbp);
1804
1805 if (ret == -1) {
1806 fasttrap_sigsegv(p, uthread, addr);
1807 new_pc = pc;
1808 break;
1809 }
1810
1811 regs64->isf.rsp = addr;
1812 new_pc = pc + tp->ftt_size;
1813 break;
1814 }
1815
1816 case FASTTRAP_T_NOP:
1817 new_pc = pc + tp->ftt_size;
1818 break;
1819
1820 case FASTTRAP_T_JMP:
1821 case FASTTRAP_T_CALL:
1822 if (tp->ftt_code == 0) {
1823 new_pc = tp->ftt_dest;
1824 } else {
1825 user_addr_t value, addr = tp->ftt_dest;
1826
1827 if (tp->ftt_base != FASTTRAP_NOREG)
1828 addr += fasttrap_getreg(regs, tp->ftt_base);
1829 if (tp->ftt_index != FASTTRAP_NOREG)
1830 addr += fasttrap_getreg(regs, tp->ftt_index) <<
1831 tp->ftt_scale;
1832
1833 if (tp->ftt_code == 1) {
1834 /*
1835 * If there's a segment prefix for this
1836 * instruction, we'll need to check permissions
1837 * and bounds on the given selector, and adjust
1838 * the address accordingly.
1839 */
1840 if (tp->ftt_segment != FASTTRAP_SEG_NONE &&
1841 fasttrap_do_seg(tp, regs, &addr) != 0) {
1842 fasttrap_sigsegv(p, uthread, addr);
1843 new_pc = pc;
1844 break;
1845 }
1846
1847 if (fasttrap_fuword64(addr, &value) == -1) {
1848 fasttrap_sigsegv(p, uthread, addr);
1849 new_pc = pc;
1850 break;
1851 }
1852 new_pc = value;
1853 } else {
1854 new_pc = addr;
1855 }
1856 }
1857
1858 /*
1859 * If this is a call instruction, we need to push the return
1860 * address onto the stack. If this fails, we send the process
1861 * a SIGSEGV and reset the pc to emulate what would happen if
1862 * this instruction weren't traced.
1863 */
1864 if (tp->ftt_type == FASTTRAP_T_CALL) {
1865 user_addr_t addr = regs64->isf.rsp - sizeof (uint64_t);
1866 int ret = fasttrap_suword64(addr, pc + tp->ftt_size);
1867
1868 if (ret == -1) {
1869 fasttrap_sigsegv(p, uthread, addr);
1870 new_pc = pc;
1871 break;
1872 }
1873
1874 regs64->isf.rsp = addr;
1875 }
1876 break;
1877
1878 case FASTTRAP_T_COMMON:
1879 {
1880 user_addr_t addr;
1881 uint8_t scratch[2 * FASTTRAP_MAX_INSTR_SIZE + 5 + 2];
1882 uint_t i = 0;
1883
1884 /*
1885 * Generic Instruction Tracing
1886 * ---------------------------
1887 *
1888 * This is the layout of the scratch space in the user-land
1889 * thread structure for our generated instructions.
1890 *
1891 * 32-bit mode bytes
1892 * ------------------------ -----
1893 * a: <original instruction> <= 15
1894 * jmp <pc + tp->ftt_size> 5
1895 * b: <original instrction> <= 15
1896 * int T_DTRACE_RET 2
1897 * -----
1898 * <= 37
1899 *
1900 * 64-bit mode bytes
1901 * ------------------------ -----
1902 * a: <original instruction> <= 15
1903 * jmp 0(%rip) 6
1904 * <pc + tp->ftt_size> 8
1905 * b: <original instruction> <= 15
1906 * int T_DTRACE_RET 2
1907 * -----
1908 * <= 46
1909 *
1910 * The %pc is set to a, and curthread->t_dtrace_astpc is set
1911 * to b. If we encounter a signal on the way out of the
1912 * kernel, trap() will set %pc to curthread->t_dtrace_astpc
1913 * so that we execute the original instruction and re-enter
1914 * the kernel rather than redirecting to the next instruction.
1915 *
1916 * If there are return probes (so we know that we're going to
1917 * need to reenter the kernel after executing the original
1918 * instruction), the scratch space will just contain the
1919 * original instruction followed by an interrupt -- the same
1920 * data as at b.
1921 *
1922 * %rip-relative Addressing
1923 * ------------------------
1924 *
1925 * There's a further complication in 64-bit mode due to %rip-
1926 * relative addressing. While this is clearly a beneficial
1927 * architectural decision for position independent code, it's
1928 * hard not to see it as a personal attack against the pid
1929 * provider since before there was a relatively small set of
1930 * instructions to emulate; with %rip-relative addressing,
1931 * almost every instruction can potentially depend on the
1932 * address at which it's executed. Rather than emulating
1933 * the broad spectrum of instructions that can now be
1934 * position dependent, we emulate jumps and others as in
1935 * 32-bit mode, and take a different tack for instructions
1936 * using %rip-relative addressing.
1937 *
1938 * For every instruction that uses the ModRM byte, the
1939 * in-kernel disassembler reports its location. We use the
1940 * ModRM byte to identify that an instruction uses
1941 * %rip-relative addressing and to see what other registers
1942 * the instruction uses. To emulate those instructions,
1943 * we modify the instruction to be %rax-relative rather than
1944 * %rip-relative (or %rcx-relative if the instruction uses
1945 * %rax; or %r8- or %r9-relative if the REX.B is present so
1946 * we don't have to rewrite the REX prefix). We then load
1947 * the value that %rip would have been into the scratch
1948 * register and generate an instruction to reset the scratch
1949 * register back to its original value. The instruction
1950 * sequence looks like this:
1951 *
1952 * 64-mode %rip-relative bytes
1953 * ------------------------ -----
1954 * a: <modified instruction> <= 15
1955 * movq $<value>, %<scratch> 6
1956 * jmp 0(%rip) 6
1957 * <pc + tp->ftt_size> 8
1958 * b: <modified instruction> <= 15
1959 * int T_DTRACE_RET 2
1960 * -----
1961 * 52
1962 *
1963 * We set curthread->t_dtrace_regv so that upon receiving
1964 * a signal we can reset the value of the scratch register.
1965 */
1966
1967 addr = uthread->t_dtrace_scratch->addr;
1968
1969 if (addr == 0LL) {
1970 fasttrap_sigtrap(p, uthread, pc); // Should be killing target proc
1971 new_pc = pc;
1972 break;
1973 }
1974
1975 ASSERT(tp->ftt_size < FASTTRAP_MAX_INSTR_SIZE);
1976
1977 uthread->t_dtrace_scrpc = addr;
1978 bcopy(tp->ftt_instr, &scratch[i], tp->ftt_size);
1979 i += tp->ftt_size;
1980
1981 if (tp->ftt_ripmode != 0) {
1982 uint64_t* reg;
1983
1984 ASSERT(tp->ftt_ripmode &
1985 (FASTTRAP_RIP_1 | FASTTRAP_RIP_2));
1986
1987 /*
1988 * If this was a %rip-relative instruction, we change
1989 * it to be either a %rax- or %rcx-relative
1990 * instruction (depending on whether those registers
1991 * are used as another operand; or %r8- or %r9-
1992 * relative depending on the value of REX.B). We then
1993 * set that register and generate a movq instruction
1994 * to reset the value.
1995 */
1996 if (tp->ftt_ripmode & FASTTRAP_RIP_X)
1997 scratch[i++] = FASTTRAP_REX(1, 0, 0, 1);
1998 else
1999 scratch[i++] = FASTTRAP_REX(1, 0, 0, 0);
2000
2001 if (tp->ftt_ripmode & FASTTRAP_RIP_1)
2002 scratch[i++] = FASTTRAP_MOV_EAX;
2003 else
2004 scratch[i++] = FASTTRAP_MOV_ECX;
2005
2006 switch (tp->ftt_ripmode) {
2007 case FASTTRAP_RIP_1:
2008 reg = &regs64->rax;
2009 uthread->t_dtrace_reg = REG_RAX;
2010 break;
2011 case FASTTRAP_RIP_2:
2012 reg = &regs64->rcx;
2013 uthread->t_dtrace_reg = REG_RCX;
2014 break;
2015 case FASTTRAP_RIP_1 | FASTTRAP_RIP_X:
2016 reg = &regs64->r8;
2017 uthread->t_dtrace_reg = REG_R8;
2018 break;
2019 case FASTTRAP_RIP_2 | FASTTRAP_RIP_X:
2020 reg = &regs64->r9;
2021 uthread->t_dtrace_reg = REG_R9;
2022 break;
2023 default:
2024 reg = NULL;
2025 panic("unhandled ripmode in fasttrap_pid_probe64");
2026 }
2027
2028 *(uint64_t *)&scratch[i] = *reg;
2029 uthread->t_dtrace_regv = *reg;
2030 *reg = pc + tp->ftt_size;
2031 i += sizeof (uint64_t);
2032 }
2033
2034 /*
2035 * Generate the branch instruction to what would have
2036 * normally been the subsequent instruction. In 32-bit mode,
2037 * this is just a relative branch; in 64-bit mode this is a
2038 * %rip-relative branch that loads the 64-bit pc value
2039 * immediately after the jmp instruction.
2040 */
2041 scratch[i++] = FASTTRAP_GROUP5_OP;
2042 scratch[i++] = FASTTRAP_MODRM(0, 4, 5);
2043 *(uint32_t *)&scratch[i] = 0;
2044 i += sizeof (uint32_t);
2045 *(uint64_t *)&scratch[i] = pc + tp->ftt_size;
2046 i += sizeof (uint64_t);
2047
2048 uthread->t_dtrace_astpc = addr + i;
2049 bcopy(tp->ftt_instr, &scratch[i], tp->ftt_size);
2050 i += tp->ftt_size;
2051 scratch[i++] = FASTTRAP_INT;
2052 scratch[i++] = T_DTRACE_RET;
2053
2054 if (fasttrap_copyout(scratch, addr, i)) {
2055 fasttrap_sigtrap(p, uthread, pc);
2056 new_pc = pc;
2057 break;
2058 }
2059
2060 if (tp->ftt_retids != NULL) {
2061 uthread->t_dtrace_step = 1;
2062 uthread->t_dtrace_ret = 1;
2063 new_pc = uthread->t_dtrace_astpc;
2064 } else {
2065 new_pc = uthread->t_dtrace_scrpc;
2066 }
2067
2068 uthread->t_dtrace_pc = pc;
2069 uthread->t_dtrace_npc = pc + tp->ftt_size;
2070 uthread->t_dtrace_on = 1;
2071 break;
2072 }
2073
2074 default:
2075 panic("fasttrap: mishandled an instruction");
2076 }
2077
2078 done:
2079 /*
2080 * APPLE NOTE:
2081 *
2082 * We're setting this earlier than Solaris does, to get a "correct"
2083 * ustack() output. In the Sun code, a() -> b() -> c() -> d() is
2084 * reported at: d, b, a. The new way gives c, b, a, which is closer
2085 * to correct, as the return instruction has already exectued.
2086 */
2087 regs64->isf.rip = new_pc;
2088
2089
2090 /*
2091 * If there were no return probes when we first found the tracepoint,
2092 * we should feel no obligation to honor any return probes that were
2093 * subsequently enabled -- they'll just have to wait until the next
2094 * time around.
2095 */
2096 if (tp->ftt_retids != NULL) {
2097 /*
2098 * We need to wait until the results of the instruction are
2099 * apparent before invoking any return probes. If this
2100 * instruction was emulated we can just call
2101 * fasttrap_return_common(); if it needs to be executed, we
2102 * need to wait until the user thread returns to the kernel.
2103 */
2104 if (tp->ftt_type != FASTTRAP_T_COMMON) {
2105 fasttrap_return_common(regs, pc, pid, new_pc);
2106 } else {
2107 ASSERT(uthread->t_dtrace_ret != 0);
2108 ASSERT(uthread->t_dtrace_pc == pc);
2109 ASSERT(uthread->t_dtrace_scrpc != 0);
2110 ASSERT(new_pc == uthread->t_dtrace_astpc);
2111 }
2112 }
2113
2114 return (0);
2115 }
2116
2117 int
2118 fasttrap_pid_probe(x86_saved_state_t *regs)
2119 {
2120 if (is_saved_state64(regs))
2121 return fasttrap_pid_probe64(regs);
2122
2123 return fasttrap_pid_probe32(regs);
2124 }
2125
2126 int
2127 fasttrap_return_probe(x86_saved_state_t *regs)
2128 {
2129 x86_saved_state64_t *regs64;
2130 x86_saved_state32_t *regs32;
2131 unsigned int p_model;
2132
2133 if (is_saved_state64(regs)) {
2134 regs64 = saved_state64(regs);
2135 regs32 = NULL;
2136 p_model = DATAMODEL_LP64;
2137 } else {
2138 regs64 = NULL;
2139 regs32 = saved_state32(regs);
2140 p_model = DATAMODEL_ILP32;
2141 }
2142
2143 proc_t *p = current_proc();
2144 uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
2145 user_addr_t pc = uthread->t_dtrace_pc;
2146 user_addr_t npc = uthread->t_dtrace_npc;
2147
2148 uthread->t_dtrace_pc = 0;
2149 uthread->t_dtrace_npc = 0;
2150 uthread->t_dtrace_scrpc = 0;
2151 uthread->t_dtrace_astpc = 0;
2152
2153 /*
2154 * Treat a child created by a call to vfork(2) as if it were its
2155 * parent. We know that there's only one thread of control in such a
2156 * process: this one.
2157 */
2158 /*
2159 * APPLE NOTE: Terry says: "You need to hold the process locks (currently: kernel funnel) for this traversal"
2160 * How do we assert this?
2161 */
2162 while (p->p_lflag & P_LINVFORK) {
2163 p = p->p_pptr;
2164 }
2165
2166 /*
2167 * We set rp->r_pc to the address of the traced instruction so
2168 * that it appears to dtrace_probe() that we're on the original
2169 * instruction, and so that the user can't easily detect our
2170 * complex web of lies. dtrace_return_probe() (our caller)
2171 * will correctly set %pc after we return.
2172 */
2173 if (p_model == DATAMODEL_LP64)
2174 regs64->isf.rip = pc;
2175 else
2176 regs32->eip = pc;
2177
2178 fasttrap_return_common(regs, pc, p->p_pid, npc);
2179
2180 return (0);
2181 }
2182
2183 uint64_t
2184 fasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno,
2185 int aframes)
2186 {
2187 #pragma unused(arg, id, parg, aframes)
2188 return (fasttrap_anarg((x86_saved_state_t *)find_user_regs(current_thread()), 1, argno));
2189 }
2190
2191 uint64_t
2192 fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno,
2193 int aframes)
2194 {
2195 #pragma unused(arg, id, parg, aframes)
2196 return (fasttrap_anarg((x86_saved_state_t *)find_user_regs(current_thread()), 0, argno));
2197 }
2198
2199 /*
2200 * APPLE NOTE: See comments by regmap array definition. We are cheating
2201 * when returning 32 bit registers.
2202 */
2203 static user_addr_t
2204 fasttrap_getreg(x86_saved_state_t *regs, uint_t reg)
2205 {
2206 if (is_saved_state64(regs)) {
2207 x86_saved_state64_t *regs64 = saved_state64(regs);
2208
2209 switch (reg) {
2210 case REG_RAX: return regs64->rax;
2211 case REG_RCX: return regs64->rcx;
2212 case REG_RDX: return regs64->rdx;
2213 case REG_RBX: return regs64->rbx;
2214 case REG_RSP: return regs64->isf.rsp;
2215 case REG_RBP: return regs64->rbp;
2216 case REG_RSI: return regs64->rsi;
2217 case REG_RDI: return regs64->rdi;
2218 case REG_R8: return regs64->r8;
2219 case REG_R9: return regs64->r9;
2220 case REG_R10: return regs64->r10;
2221 case REG_R11: return regs64->r11;
2222 case REG_R12: return regs64->r12;
2223 case REG_R13: return regs64->r13;
2224 case REG_R14: return regs64->r14;
2225 case REG_R15: return regs64->r15;
2226 }
2227
2228 panic("dtrace: unhandled x86_64 getreg() constant");
2229 } else {
2230 x86_saved_state32_t *regs32 = saved_state32(regs);
2231
2232 switch (reg) {
2233 case REG_RAX: return regs32->eax;
2234 case REG_RCX: return regs32->ecx;
2235 case REG_RDX: return regs32->edx;
2236 case REG_RBX: return regs32->ebx;
2237 case REG_RSP: return regs32->uesp;
2238 case REG_RBP: return regs32->ebp;
2239 case REG_RSI: return regs32->esi;
2240 case REG_RDI: return regs32->edi;
2241 }
2242
2243 panic("dtrace: unhandled i386 getreg() constant");
2244 }
2245
2246 return 0;
2247 }
mirror of XNU