+
+#if !defined(RC_HIDE_XNU_J137)
+/*
+ * If a thread is using an AVX-sized savearea:
+ * - allocate a new AVX512-sized area,
+ * - copy the 256-bit state into the 512-bit area,
+ * - deallocate the smaller area
+ */
+static void
+fpu_savearea_promote_avx512(thread_t thread)
+{
+ struct x86_avx_thread_state *ifps = NULL;
+ struct x86_avx512_thread_state *ifps512 = NULL;
+ pcb_t pcb = THREAD_TO_PCB(thread);
+ boolean_t do_avx512_alloc = FALSE;
+
+ DBG("fpu_upgrade_savearea(%p)\n", thread);
+
+ simple_lock(&pcb->lock, LCK_GRP_NULL);
+
+ ifps = pcb->ifps;
+ if (ifps == NULL) {
+ pcb->xstate = AVX512;
+ simple_unlock(&pcb->lock);
+ if (thread != current_thread()) {
+ /* nothing to be done */
+
+ return;
+ }
+ fpnoextflt();
+ return;
+ }
+
+ if (pcb->xstate != AVX512) {
+ do_avx512_alloc = TRUE;
+ }
+ simple_unlock(&pcb->lock);
+
+ if (do_avx512_alloc == TRUE) {
+ ifps512 = fp_state_alloc(AVX512);
+ }
+
+ simple_lock(&pcb->lock, LCK_GRP_NULL);
+ if (thread == current_thread()) {
+ boolean_t intr;
+
+ intr = ml_set_interrupts_enabled(FALSE);
+
+ clear_ts();
+ fp_save(thread);
+ clear_fpu();
+
+ xsetbv(0, AVX512_XMASK);
+ current_cpu_datap()->cpu_xstate = AVX512;
+ (void)ml_set_interrupts_enabled(intr);
+ }
+ assert(ifps->fp.fp_valid);
+
+ /* Allocate an AVX512 savearea and copy AVX state into it */
+ if (pcb->xstate != AVX512) {
+ bcopy(ifps, ifps512, fp_state_size[AVX]);
+ pcb->ifps = ifps512;
+ pcb->xstate = AVX512;
+ ifps512 = NULL;
+ } else {
+ ifps = NULL;
+ }
+ /* The PCB lock is redundant in some scenarios given the higher level
+ * thread mutex, but its pre-emption disablement is relied upon here
+ */
+ simple_unlock(&pcb->lock);
+
+ if (ifps) {
+ fp_state_free(ifps, AVX);
+ }
+ if (ifps512) {
+ fp_state_free(ifps, AVX512);
+ }
+}
+
+/*
+ * Upgrade the calling thread to AVX512.
+ */
+boolean_t
+fpu_thread_promote_avx512(thread_t thread)
+{
+ task_t task = current_task();
+
+ if (thread != current_thread()) {
+ return FALSE;
+ }
+ if (!ml_fpu_avx512_enabled()) {
+ return FALSE;
+ }
+
+ fpu_savearea_promote_avx512(thread);
+
+ /* Racy but the task's xstate is only a hint */
+ task->xstate = AVX512;
+
+ return TRUE;
+}
+
+
+/*
+ * Called from user_trap() when an invalid opcode fault is taken.
+ * If the user is attempting an AVX512 instruction on a machine
+ * that supports this, we switch the calling thread to use
+ * a larger savearea, set its XCR0 bit mask to enable AVX512 and
+ * return directly via thread_exception_return().
+ * Otherwise simply return.
+ */
+#define MAX_X86_INSN_LENGTH (16)
+void
+fpUDflt(user_addr_t rip)
+{
+ uint8_t instruction_prefix;
+ boolean_t is_AVX512_instruction = FALSE;
+ user_addr_t original_rip = rip;
+ do {
+ /* TODO: as an optimisation, copy up to the lesser of the
+ * next page boundary or maximal prefix length in one pass
+ * rather than issue multiple copyins
+ */
+ if (copyin(rip, (char *) &instruction_prefix, 1)) {
+ return;
+ }
+ DBG("fpUDflt(0x%016llx) prefix: 0x%x\n",
+ rip, instruction_prefix);
+ /* TODO: determine more specifically which prefixes
+ * are sane possibilities for AVX512 insns
+ */
+ switch (instruction_prefix) {
+ case 0x2E: /* CS segment override */
+ case 0x36: /* SS segment override */
+ case 0x3E: /* DS segment override */
+ case 0x26: /* ES segment override */
+ case 0x64: /* FS segment override */
+ case 0x65: /* GS segment override */
+ case 0x66: /* Operand-size override */
+ case 0x67: /* address-size override */
+ /* Skip optional prefixes */
+ rip++;
+ if ((rip - original_rip) > MAX_X86_INSN_LENGTH) {
+ return;
+ }
+ break;
+ case 0x62: /* EVEX */
+ case 0xC5: /* VEX 2-byte */
+ case 0xC4: /* VEX 3-byte */
+ is_AVX512_instruction = TRUE;
+ break;
+ default:
+ return;
+ }
+ } while (!is_AVX512_instruction);
+
+ /* Here if we detect attempted execution of an AVX512 instruction */
+
+ /*
+ * Fail if this machine doesn't support AVX512
+ */
+ if (fpu_capability != AVX512) {
+ return;
+ }
+
+ assert(xgetbv(XCR0) == AVX_XMASK);
+
+ DBG("fpUDflt() switching xstate to AVX512\n");
+ (void) fpu_thread_promote_avx512(current_thread());
+
+ thread_exception_return();
+ /* NOT REACHED */
+}
+#endif /* !defined(RC_HIDE_XNU_J137) */
+
projects / apple / xnu.git / blobdiff