+#endif
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Walloca"
+ stack_buffer = __builtin_alloca(insn_copyin_count);
+#pragma clang diagnostic pop
+
+ if (rip >= (insn_copyin_count / 2)) {
+ start_addr = rip - (insn_copyin_count / 2);
+ } else {
+ start_addr = 0;
+ }
+
+ if (start_addr < rip_page) {
+ insn_offset = (insn_copyin_count / 2) - (rip_page - start_addr);
+ end_addr += (rip_page - start_addr);
+ start_addr = rip_page;
+ } else if (end_addr >= (rip_page + (~pagemask + 1))) {
+ start_addr -= (end_addr - (rip_page + (~pagemask + 1))); /* Adjust start address backward */
+ /* Adjust instruction offset due to start address change */
+ insn_offset = (insn_copyin_count / 2) + (end_addr - (rip_page + (~pagemask + 1)));
+ end_addr = rip_page + (~pagemask + 1); /* clip to the start of the next page (non-inclusive */
+ } else {
+ insn_offset = insn_copyin_count / 2;
+ }
+
+ disable_preemption(); /* Prevent copyin from faulting in the instruction stream */
+ if (
+#if DEVELOPMENT || DEBUG
+ (insnstream_force_cacheline_mismatch < 2) &&
+#endif
+ ((end_addr > start_addr) && (copyin_err = copyin(start_addr, stack_buffer, end_addr - start_addr)) == 0)) {
+ enable_preemption();
+
+ if (pcb->insn_state == 0) {
+ pcb->insn_state = kalloc(sizeof(x86_instruction_state_t));
+ }
+
+ if (pcb->insn_state != 0) {
+ bcopy(stack_buffer, pcb->insn_state->insn_bytes, end_addr - start_addr);
+ bzero(&pcb->insn_state->insn_bytes[end_addr - start_addr],
+ insn_copyin_count - (end_addr - start_addr));
+
+ pcb->insn_state->insn_stream_valid_bytes = (int)(end_addr - start_addr);
+ pcb->insn_state->insn_offset = (int)insn_offset;
+
+#if DEVELOPMENT || DEBUG
+ /* Now try to validate the cacheline we read at early-fault time matches the code
+ * copied in. Before we do that, we have to make sure the buffer contains a valid
+ * cacheline by looking for the 2 sentinel values written in the event the cacheline
+ * could not be copied.
+ */
+#define CACHELINE_DATA_NOT_PRESENT 0xdeadc0debeefcafeULL
+#define CACHELINE_MASK (CACHELINE_SIZE - 1)
+
+ if (inspect_cacheline &&
+ (*(uint64_t *)(uintptr_t)&pcb->insn_cacheline[0] != CACHELINE_DATA_NOT_PRESENT &&
+ *(uint64_t *)(uintptr_t)&pcb->insn_cacheline[8] != CACHELINE_DATA_NOT_PRESENT)) {
+ /*
+ * The position of the cacheline in the instruction buffer is at offset
+ * insn_offset - (rip & CACHELINE_MASK)
+ */
+ if (__improbable((rip & CACHELINE_MASK) > insn_offset)) {
+ printf("thread %p code cacheline @ %p clipped wrt copied-in code (offset %d)\n",
+ thread, (void *)(rip & ~CACHELINE_MASK), (int)(rip & CACHELINE_MASK));
+ } else if (bcmp(&pcb->insn_state->insn_bytes[insn_offset - (rip & CACHELINE_MASK)],
+ &pcb->insn_cacheline[0], CACHELINE_SIZE) != 0
+ || insnstream_force_cacheline_mismatch
+ ) {
+#if x86_INSTRUCTION_STATE_CACHELINE_SIZE != CACHELINE_SIZE
+#error cacheline size mismatch
+#endif
+ bcopy(&pcb->insn_cacheline[0], &pcb->insn_state->insn_cacheline[0],
+ x86_INSTRUCTION_STATE_CACHELINE_SIZE);
+ /* Mark the instruction stream as being out-of-synch */
+ pcb->insn_state->out_of_synch = 1;
+
+ printf("thread %p code cacheline @ %p mismatches with copied-in code [trap 0x%x]\n",
+ thread, (void *)(rip & ~CACHELINE_MASK), trap_code);
+ for (int i = 0; i < 8; i++) {
+ printf("\t[%d] cl=0x%08llx vs. ci=0x%08llx\n", i, *(uint64_t *)(uintptr_t)&pcb->insn_cacheline[i * 8],
+ *(uint64_t *)(uintptr_t)&pcb->insn_state->insn_bytes[(i * 8) + insn_offset - (rip & CACHELINE_MASK)]);
+ }
+ if (panic_on_cacheline_mismatch) {
+ panic("Cacheline mismatch while processing unhandled exception.");
+ }
+ } else {
+ printf("thread %p code cacheline @ %p DOES match with copied-in code\n",
+ thread, (void *)(rip & ~CACHELINE_MASK));
+ pcb->insn_state->out_of_synch = 0;
+ }
+ } else if (inspect_cacheline) {
+ printf("thread %p could not capture code cacheline at fault IP %p [offset %d]\n",
+ (void *)thread, (void *)rip, (int)(insn_offset - (rip & CACHELINE_MASK)));
+ pcb->insn_state->out_of_synch = 0;
+ }
+#else
+ pcb->insn_state->out_of_synch = 0;
+#endif /* DEVELOPMENT || DEBUG */
+
+#if defined(MACH_BSD) && (DEVELOPMENT || DEBUG)
+ if (panic_on_trap_procname[0] != 0) {
+ char procnamebuf[65] = {0};
+
+ if (thread->task->bsd_info != NULL) {
+ procname = proc_name_address(thread->task->bsd_info);
+ strlcpy(procnamebuf, procname, sizeof(procnamebuf));
+
+ if (strcasecmp(panic_on_trap_procname, procnamebuf) == 0 &&
+ ((1U << trap_code) & panic_on_trap_mask) != 0) {
+ panic("Panic requested on trap type 0x%x for process `%s'", trap_code,
+ panic_on_trap_procname);
+ /*NORETURN*/
+ }
+ }
+ }
+#endif /* MACH_BSD && (DEVELOPMENT || DEBUG) */
+ }
+ } else {
+ enable_preemption();
+
+ pcb->insn_state_copyin_failure_errorcode = copyin_err;
+#if DEVELOPMENT || DEBUG
+ if (inspect_cacheline && pcb->insn_state == 0) {
+ pcb->insn_state = kalloc(sizeof(x86_instruction_state_t));
+ }
+ if (pcb->insn_state != 0) {
+ pcb->insn_state->insn_stream_valid_bytes = 0;
+ pcb->insn_state->insn_offset = 0;
+
+ if (inspect_cacheline &&
+ (*(uint64_t *)(uintptr_t)&pcb->insn_cacheline[0] != CACHELINE_DATA_NOT_PRESENT &&
+ *(uint64_t *)(uintptr_t)&pcb->insn_cacheline[8] != CACHELINE_DATA_NOT_PRESENT)) {
+ /*
+ * We can still copy the cacheline into the instruction state structure
+ * if it contains valid data
+ */
+ pcb->insn_state->out_of_synch = 1;
+ bcopy(&pcb->insn_cacheline[0], &pcb->insn_state->insn_cacheline[0],
+ x86_INSTRUCTION_STATE_CACHELINE_SIZE);
+ }
+ }
+#endif /* DEVELOPMENT || DEBUG */
+ }
projects / apple / xnu.git / blobdiff