osfmk/i386/fpu.c

   1 /*
   2  * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1992-1990 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58
  59 #include <platforms.h>
  60
  61 #include <mach/exception_types.h>
  62 #include <mach/i386/thread_status.h>
  63 #include <mach/i386/fp_reg.h>
  64 #include <mach/branch_predicates.h>
  65
  66 #include <kern/mach_param.h>
  67 #include <kern/processor.h>
  68 #include <kern/thread.h>
  69 #include <kern/zalloc.h>
  70 #include <kern/misc_protos.h>
  71 #include <kern/spl.h>
  72 #include <kern/assert.h>
  73
  74 #include <libkern/OSAtomic.h>
  75
  76 #include <architecture/i386/pio.h>
  77 #include <i386/cpuid.h>
  78 #include <i386/fpu.h>
  79 #include <i386/proc_reg.h>
  80 #include <i386/misc_protos.h>
  81 #include <i386/thread.h>
  82 #include <i386/trap.h>
  83
  84 int             fp_kind = FP_NO;        /* not inited */
  85 zone_t          ifps_zone;              /* zone for FPU save area */
  86
  87 #define ALIGNED(addr,size)      (((uintptr_t)(addr)&((size)-1))==0)
  88
  89 /* Forward */
  90
  91 extern void             fpinit(void);
  92 extern void             fp_save(
  93                                 thread_t        thr_act);
  94 extern void             fp_load(
  95                                 thread_t        thr_act);
  96
  97 static void configure_mxcsr_capability_mask(struct x86_avx_thread_state *fps);
  98
  99 struct x86_avx_thread_state initial_fp_state __attribute((aligned(64)));
 100
 101
 102 /* Global MXCSR capability bitmask */
 103 static unsigned int mxcsr_capability_mask;
 104
 105 #define fninit() \
 106         __asm__ volatile("fninit")
 107
 108 #define fnstcw(control) \
 109         __asm__("fnstcw %0" : "=m" (*(unsigned short *)(control)))
 110
 111 #define fldcw(control) \
 112         __asm__ volatile("fldcw %0" : : "m" (*(unsigned short *) &(control)) )
 113
 114 #define fnclex() \
 115         __asm__ volatile("fnclex")
 116
 117 #define fnsave(state)  \
 118         __asm__ volatile("fnsave %0" : "=m" (*state))
 119
 120 #define frstor(state) \
 121         __asm__ volatile("frstor %0" : : "m" (state))
 122
 123 #define fwait() \
 124         __asm__("fwait");
 125
 126 #define fxrstor(addr)           __asm__ __volatile__("fxrstor %0" : : "m" (*(addr)))
 127 #define fxsave(addr)            __asm__ __volatile__("fxsave %0" : "=m" (*(addr)))
 128
 129 static uint32_t fp_register_state_size = 0;
 130 static uint32_t fpu_YMM_present = FALSE;
 131 static uint32_t cpuid_reevaluated = 0;
 132
 133 static void fpu_store_registers(void *, boolean_t);
 134 static void fpu_load_registers(void *);
 135
 136 extern  void xsave64o(void);
 137 extern  void xrstor64o(void);
 138
 139 #define XMASK ((uint32_t) (XFEM_X87 | XFEM_SSE | XFEM_YMM))
 140
 141 /* DRK: TODO replace opcodes with mnemonics when assembler support available */
 142
 143 static inline void xsetbv(uint32_t mask_hi, uint32_t mask_lo) {
 144         __asm__ __volatile__(".short 0x010F\n\t.byte 0xD1" :: "a"(mask_lo), "d"(mask_hi), "c" (XCR0));
 145 }
 146
 147 static inline void xsave(void *a) {
 148         /* MOD 0x4, operand ECX 0x1 */
 149         __asm__ __volatile__(".short 0xAE0F\n\t.byte 0x21" :: "a"(XMASK), "d"(0), "c" (a));
 150 }
 151
 152 static inline void xrstor(void *a) {
 153         /* MOD 0x5, operand ECX 0x1 */
 154         __asm__ __volatile__(".short 0xAE0F\n\t.byte 0x29" :: "a"(XMASK), "d"(0), "c" (a));
 155 }
 156
 157 static inline void xsave64(void *a) {
 158         /* Out of line call that executes in 64-bit mode on K32 */
 159         __asm__ __volatile__("call _xsave64o" :: "a"(XMASK), "d"(0), "c" (a));
 160 }
 161
 162 static inline void xrstor64(void *a) {
 163         /* Out of line call that executes in 64-bit mode on K32 */
 164         __asm__ __volatile__("call _xrstor64o" :: "a"(XMASK), "d"(0), "c" (a));
 165 }
 166
 167 static inline unsigned short
 168 fnstsw(void)
 169 {
 170         unsigned short status;
 171         __asm__ volatile("fnstsw %0" : "=ma" (status));
 172         return(status);
 173 }
 174
 175 /*
 176  * Configure the initial FPU state presented to new threads.
 177  * Determine the MXCSR capability mask, which allows us to mask off any
 178  * potentially unsafe "reserved" bits before restoring the FPU context.
 179  * *Not* per-cpu, assumes symmetry.
 180  */
 181
 182 static void
 183 configure_mxcsr_capability_mask(struct x86_avx_thread_state *fps)
 184 {
 185         /* XSAVE requires a 64 byte aligned store */
 186         assert(ALIGNED(fps, 64));
 187         /* Clear, to prepare for the diagnostic FXSAVE */
 188         bzero(fps, sizeof(*fps));
 189
 190         fpinit();
 191         fpu_store_registers(fps, FALSE);
 192
 193         mxcsr_capability_mask = fps->fx_MXCSR_MASK;
 194
 195         /* Set default mask value if necessary */
 196         if (mxcsr_capability_mask == 0)
 197                 mxcsr_capability_mask = 0xffbf;
 198
 199         /* Clear vector register store */
 200         bzero(&fps->fx_XMM_reg[0][0], sizeof(fps->fx_XMM_reg));
 201         bzero(&fps->x_YMMH_reg[0][0], sizeof(fps->x_YMMH_reg));
 202
 203         fps->fp_valid = TRUE;
 204         fps->fp_save_layout = fpu_YMM_present ? XSAVE32: FXSAVE32;
 205         fpu_load_registers(fps);
 206
 207         /* Poison values to trap unsafe usage */
 208         fps->fp_valid = 0xFFFFFFFF;
 209         fps->fp_save_layout = FP_UNUSED;
 210
 211         /* Re-enable FPU/SSE DNA exceptions */
 212         set_ts();
 213 }
 214
 215
 216 /*
 217  * Look for FPU and initialize it.
 218  * Called on each CPU.
 219  */
 220 void
 221 init_fpu(void)
 222 {
 223 #if     DEBUG
 224         unsigned short  status;
 225         unsigned short  control;
 226 #endif
 227         /*
 228          * Check for FPU by initializing it,
 229          * then trying to read the correct bit patterns from
 230          * the control and status registers.
 231          */
 232         set_cr0((get_cr0() & ~(CR0_EM|CR0_TS)) | CR0_NE);       /* allow use of FPU */
 233         fninit();
 234 #if     DEBUG
 235         status = fnstsw();
 236         fnstcw(&control);
 237
 238         assert(((status & 0xff) == 0) && ((control & 0x103f) == 0x3f));
 239 #endif
 240         /* Advertise SSE support */
 241         if (cpuid_features() & CPUID_FEATURE_FXSR) {
 242                 fp_kind = FP_FXSR;
 243                 set_cr4(get_cr4() | CR4_OSFXS);
 244                 /* And allow SIMD exceptions if present */
 245                 if (cpuid_features() & CPUID_FEATURE_SSE) {
 246                         set_cr4(get_cr4() | CR4_OSXMM);
 247                 }
 248                 fp_register_state_size = sizeof(struct x86_fx_thread_state);
 249
 250         } else
 251                 panic("fpu is not FP_FXSR");
 252
 253         /* Configure the XSAVE context mechanism if the processor supports
 254          * AVX/YMM registers
 255          */
 256         if (cpuid_features() & CPUID_FEATURE_XSAVE) {
 257                 cpuid_xsave_leaf_t *xsp = &cpuid_info()->cpuid_xsave_leaf;
 258                 if (xsp->extended_state[0] & (uint32_t)XFEM_YMM) {
 259                         assert(xsp->extended_state[0] & (uint32_t) XFEM_SSE);
 260                         /* XSAVE container size for all features */
 261                         assert(xsp->extended_state[2] == sizeof(struct x86_avx_thread_state));
 262                         fp_register_state_size = sizeof(struct x86_avx_thread_state);
 263                         fpu_YMM_present = TRUE;
 264                         set_cr4(get_cr4() | CR4_OSXSAVE);
 265                         xsetbv(0, XMASK);
 266                         /* Re-evaluate CPUID, once, to reflect OSXSAVE */
 267                         if (OSCompareAndSwap(0, 1, &cpuid_reevaluated))
 268                                 cpuid_set_info();
 269                         /* DRK: consider verifying AVX offset with cpuid(d, ECX:2) */
 270                 }
 271         }
 272         else
 273                 fpu_YMM_present = FALSE;
 274
 275         fpinit();
 276
 277         /*
 278          * Trap wait instructions.  Turn off FPU for now.
 279          */
 280         set_cr0(get_cr0() | CR0_TS | CR0_MP);
 281 }
 282
 283 /*
 284  * Allocate and initialize FP state for current thread.
 285  * Don't load state.
 286  */
 287 static void *
 288 fp_state_alloc(void)
 289 {
 290         void *ifps = zalloc(ifps_zone);
 291
 292 #if     DEBUG
 293         if (!(ALIGNED(ifps,64))) {
 294                 panic("fp_state_alloc: %p, %u, %p, %u", ifps, (unsigned) ifps_zone->elem_size, (void *) ifps_zone->free_elements, (unsigned) ifps_zone->alloc_size);
 295         }
 296 #endif
 297         return ifps;
 298 }
 299
 300 static inline void
 301 fp_state_free(void *ifps)
 302 {
 303         zfree(ifps_zone, ifps);
 304 }
 305
 306 void clear_fpu(void)
 307 {
 308         set_ts();
 309 }
 310
 311
 312 static void fpu_load_registers(void *fstate) {
 313         struct x86_fx_thread_state *ifps = fstate;
 314         fp_save_layout_t layout = ifps->fp_save_layout;
 315
 316         assert(layout == FXSAVE32 || layout == FXSAVE64 || layout == XSAVE32 || layout == XSAVE64);
 317         assert(ALIGNED(ifps, 64));
 318         assert(ml_get_interrupts_enabled() == FALSE);
 319
 320 #if     DEBUG
 321         if (layout == XSAVE32 || layout == XSAVE64) {
 322                 struct x86_avx_thread_state *iavx = fstate;
 323                 unsigned i;
 324                 /* Verify reserved bits in the XSAVE header*/
 325                 if (iavx->_xh.xsbv & ~7)
 326                         panic("iavx->_xh.xsbv: 0x%llx", iavx->_xh.xsbv);
 327                 for (i = 0; i < sizeof(iavx->_xh.xhrsvd); i++)
 328                         if (iavx->_xh.xhrsvd[i])
 329                                 panic("Reserved bit set");
 330         }
 331         if (fpu_YMM_present) {
 332                 if (layout != XSAVE32 && layout != XSAVE64)
 333                         panic("Inappropriate layout: %u\n", layout);
 334         }
 335 #endif  /* DEBUG */
 336
 337         if ((layout == XSAVE64) || (layout == XSAVE32))
 338                 xrstor(ifps);
 339         else
 340                 fxrstor(ifps);
 341 }
 342
 343 static void fpu_store_registers(void *fstate, boolean_t is64) {
 344         struct x86_fx_thread_state *ifps = fstate;
 345         assert(ALIGNED(ifps, 64));
 346         if (fpu_YMM_present) {
 347                 xsave(ifps);
 348                 ifps->fp_save_layout = is64 ? XSAVE64 : XSAVE32;
 349         }
 350         else {
 351                 fxsave(ifps);
 352                 ifps->fp_save_layout = is64 ? FXSAVE64 : FXSAVE32;
 353         }
 354 }
 355
 356 /*
 357  * Initialize FP handling.
 358  */
 359
 360 void
 361 fpu_module_init(void)
 362 {
 363         if ((fp_register_state_size != sizeof(struct x86_fx_thread_state)) &&
 364             (fp_register_state_size != sizeof(struct x86_avx_thread_state)))
 365                 panic("fpu_module_init: incorrect savearea size %u\n", fp_register_state_size);
 366
 367         assert(fpu_YMM_present != 0xFFFFFFFF);
 368
 369         /* We explicitly choose an allocation size of 64
 370          * to eliminate waste for the 832 byte sized
 371          * AVX XSAVE register save area.
 372          */
 373         ifps_zone = zinit(fp_register_state_size,
 374                           thread_max * fp_register_state_size,
 375                           64 * fp_register_state_size,
 376                           "x86 fpsave state");
 377
 378         /* To maintain the required alignment, disable
 379          * zone debugging for this zone as that appends
 380          * 16 bytes to each element.
 381          */
 382         zone_change(ifps_zone, Z_ALIGNMENT_REQUIRED, TRUE);
 383         /* Determine MXCSR reserved bits and configure initial FPU state*/
 384         configure_mxcsr_capability_mask(&initial_fp_state);
 385 }
 386
 387 /*
 388  * Save thread`s FPU context.
 389  */
 390 void
 391 fpu_save_context(thread_t thread)
 392 {
 393         struct x86_fx_thread_state *ifps;
 394
 395         assert(ml_get_interrupts_enabled() == FALSE);
 396         ifps = (thread)->machine.ifps;
 397 #if     DEBUG
 398         if (ifps && ((ifps->fp_valid != FALSE) && (ifps->fp_valid != TRUE))) {
 399                 panic("ifps->fp_valid: %u\n", ifps->fp_valid);
 400         }
 401 #endif
 402         if (ifps != 0 && (ifps->fp_valid == FALSE)) {
 403                 /* Clear CR0.TS in preparation for the FP context save. In
 404                  * theory, this shouldn't be necessary since a live FPU should
 405                  * indicate that TS is clear. However, various routines
 406                  * (such as sendsig & sigreturn) manipulate TS directly.
 407                  */
 408                 clear_ts();
 409                 /* registers are in FPU - save to memory */
 410                 fpu_store_registers(ifps, (thread_is_64bit(thread) && is_saved_state64(thread->machine.iss)));
 411                 ifps->fp_valid = TRUE;
 412         }
 413         set_ts();
 414 }
 415
 416
 417 /*
 418  * Free a FPU save area.
 419  * Called only when thread terminating - no locking necessary.
 420  */
 421 void
 422 fpu_free(void *fps)
 423 {
 424         fp_state_free(fps);
 425 }
 426
 427 /*
 428  * Set the floating-point state for a thread based
 429  * on the FXSave formatted data. This is basically
 430  * the same as fpu_set_state except it uses the
 431  * expanded data structure.
 432  * If the thread is not the current thread, it is
 433  * not running (held).  Locking needed against
 434  * concurrent fpu_set_state or fpu_get_state.
 435  */
 436 kern_return_t
 437 fpu_set_fxstate(
 438         thread_t        thr_act,
 439         thread_state_t  tstate,
 440         thread_flavor_t f)
 441 {
 442         struct x86_fx_thread_state *ifps;
 443         struct x86_fx_thread_state *new_ifps;
 444         x86_float_state64_t     *state;
 445         pcb_t   pcb;
 446         size_t  state_size = sizeof(struct x86_fx_thread_state);
 447         boolean_t       old_valid;
 448         if (fp_kind == FP_NO)
 449             return KERN_FAILURE;
 450
 451         if ((f == x86_AVX_STATE32 || f == x86_AVX_STATE64) &&
 452             !ml_fpu_avx_enabled())
 453             return KERN_FAILURE;
 454
 455         state = (x86_float_state64_t *)tstate;
 456
 457         assert(thr_act != THREAD_NULL);
 458         pcb = THREAD_TO_PCB(thr_act);
 459
 460         if (state == NULL) {
 461             /*
 462              * new FPU state is 'invalid'.
 463              * Deallocate the fp state if it exists.
 464              */
 465             simple_lock(&pcb->lock);
 466
 467                 ifps = pcb->ifps;
 468                 pcb->ifps = 0;
 469
 470             simple_unlock(&pcb->lock);
 471
 472                 if (ifps != 0)
 473                 fp_state_free(ifps);
 474         } else {
 475             /*
 476              * Valid state.  Allocate the fp state if there is none.
 477              */
 478             new_ifps = 0;
 479         Retry:
 480             simple_lock(&pcb->lock);
 481
 482                 ifps = pcb->ifps;
 483             if (ifps == 0) {
 484                 if (new_ifps == 0) {
 485                     simple_unlock(&pcb->lock);
 486                     new_ifps = fp_state_alloc();
 487                     goto Retry;
 488                 }
 489                 ifps = new_ifps;
 490                 new_ifps = 0;
 491                 pcb->ifps = ifps;
 492             }
 493             /*
 494              * now copy over the new data.
 495              */
 496             old_valid = ifps->fp_valid;
 497
 498 #if     DEBUG
 499             if ((old_valid == FALSE) && (thr_act != current_thread())) {
 500                     panic("fpu_set_fxstate inconsistency, thread: %p not stopped", thr_act);
 501             }
 502 #endif
 503             /*
 504              * Clear any reserved bits in the MXCSR to prevent a GPF
 505              * when issuing an FXRSTOR.
 506              */
 507
 508             state->fpu_mxcsr &= mxcsr_capability_mask;
 509
 510             bcopy((char *)&state->fpu_fcw, (char *)ifps, state_size);
 511
 512             if (fpu_YMM_present) {
 513                 struct x86_avx_thread_state *iavx = (void *) ifps;
 514                 uint32_t fpu_nyreg = 0;
 515
 516                 if (f == x86_AVX_STATE32)
 517                         fpu_nyreg = 8;
 518                 else if (f == x86_AVX_STATE64)
 519                         fpu_nyreg = 16;
 520
 521                 if (fpu_nyreg) {
 522                         x86_avx_state64_t *ystate = (x86_avx_state64_t *) state;
 523                         bcopy(&ystate->__fpu_ymmh0, &iavx->x_YMMH_reg[0][0], fpu_nyreg * sizeof(_STRUCT_XMM_REG));
 524                 }
 525
 526                 iavx->fp_save_layout = thread_is_64bit(thr_act) ? XSAVE64 : XSAVE32;
 527                 /* Sanitize XSAVE header */
 528                 bzero(&iavx->_xh.xhrsvd[0], sizeof(iavx->_xh.xhrsvd));
 529                 if (fpu_nyreg)
 530                         iavx->_xh.xsbv = (XFEM_YMM | XFEM_SSE | XFEM_X87);
 531                 else
 532                         iavx->_xh.xsbv = (XFEM_SSE | XFEM_X87);
 533             }
 534             else
 535                 ifps->fp_save_layout = thread_is_64bit(thr_act) ? FXSAVE64 : FXSAVE32;
 536             ifps->fp_valid = old_valid;
 537
 538             if (old_valid == FALSE) {
 539                     boolean_t istate = ml_set_interrupts_enabled(FALSE);
 540                     ifps->fp_valid = TRUE;
 541                     set_ts();
 542                     ml_set_interrupts_enabled(istate);
 543             }
 544
 545             simple_unlock(&pcb->lock);
 546
 547             if (new_ifps != 0)
 548                 fp_state_free(new_ifps);
 549         }
 550         return KERN_SUCCESS;
 551 }
 552
 553 /*
 554  * Get the floating-point state for a thread.
 555  * If the thread is not the current thread, it is
 556  * not running (held).  Locking needed against
 557  * concurrent fpu_set_state or fpu_get_state.
 558  */
 559 kern_return_t
 560 fpu_get_fxstate(
 561         thread_t        thr_act,
 562         thread_state_t  tstate,
 563         thread_flavor_t f)
 564 {
 565         struct x86_fx_thread_state      *ifps;
 566         x86_float_state64_t     *state;
 567         kern_return_t   ret = KERN_FAILURE;
 568         pcb_t   pcb;
 569         size_t  state_size = sizeof(struct x86_fx_thread_state);
 570
 571         if (fp_kind == FP_NO)
 572                 return KERN_FAILURE;
 573
 574         if ((f == x86_AVX_STATE32 || f == x86_AVX_STATE64) &&
 575             !ml_fpu_avx_enabled())
 576                 return KERN_FAILURE;
 577
 578         state = (x86_float_state64_t *)tstate;
 579
 580         assert(thr_act != THREAD_NULL);
 581         pcb = THREAD_TO_PCB(thr_act);
 582
 583         simple_lock(&pcb->lock);
 584
 585         ifps = pcb->ifps;
 586         if (ifps == 0) {
 587                 /*
 588                  * No valid floating-point state.
 589                  */
 590
 591                 bcopy((char *)&initial_fp_state, (char *)&state->fpu_fcw,
 592                     state_size);
 593
 594                 simple_unlock(&pcb->lock);
 595
 596                 return KERN_SUCCESS;
 597         }
 598         /*
 599          * Make sure we`ve got the latest fp state info
 600          * If the live fpu state belongs to our target
 601          */
 602         if (thr_act == current_thread()) {
 603                 boolean_t       intr;
 604
 605                 intr = ml_set_interrupts_enabled(FALSE);
 606
 607                 clear_ts();
 608                 fp_save(thr_act);
 609                 clear_fpu();
 610
 611                 (void)ml_set_interrupts_enabled(intr);
 612         }
 613         if (ifps->fp_valid) {
 614                 bcopy((char *)ifps, (char *)&state->fpu_fcw, state_size);
 615                 if (fpu_YMM_present) {
 616                         struct x86_avx_thread_state *iavx = (void *) ifps;
 617                         uint32_t fpu_nyreg = 0;
 618
 619                         if (f == x86_AVX_STATE32)
 620                                 fpu_nyreg = 8;
 621                         else if (f == x86_AVX_STATE64)
 622                                 fpu_nyreg = 16;
 623
 624                         if (fpu_nyreg) {
 625                                 x86_avx_state64_t *ystate = (x86_avx_state64_t *) state;
 626                                 bcopy(&iavx->x_YMMH_reg[0][0], &ystate->__fpu_ymmh0, fpu_nyreg * sizeof(_STRUCT_XMM_REG));
 627                         }
 628                 }
 629
 630                 ret = KERN_SUCCESS;
 631         }
 632         simple_unlock(&pcb->lock);
 633
 634         return ret;
 635 }
 636
 637
 638
 639 /*
 640  * the child thread is 'stopped' with the thread
 641  * mutex held and is currently not known by anyone
 642  * so no way for fpu state to get manipulated by an
 643  * outside agency -> no need for pcb lock
 644  */
 645
 646 void
 647 fpu_dup_fxstate(
 648         thread_t        parent,
 649         thread_t        child)
 650 {
 651         struct x86_fx_thread_state *new_ifps = NULL;
 652         boolean_t       intr;
 653         pcb_t           ppcb;
 654
 655         ppcb = THREAD_TO_PCB(parent);
 656
 657         if (ppcb->ifps == NULL)
 658                 return;
 659
 660         if (child->machine.ifps)
 661                 panic("fpu_dup_fxstate: child's ifps non-null");
 662
 663         new_ifps = fp_state_alloc();
 664
 665         simple_lock(&ppcb->lock);
 666
 667         if (ppcb->ifps != NULL) {
 668                 struct x86_fx_thread_state *ifps = ppcb->ifps;
 669                 /*
 670                  * Make sure we`ve got the latest fp state info
 671                  */
 672                 intr = ml_set_interrupts_enabled(FALSE);
 673                 assert(current_thread() == parent);
 674                 clear_ts();
 675                 fp_save(parent);
 676                 clear_fpu();
 677
 678                 (void)ml_set_interrupts_enabled(intr);
 679
 680                 if (ifps->fp_valid) {
 681                         child->machine.ifps = new_ifps;
 682                         assert((fp_register_state_size == sizeof(struct x86_fx_thread_state)) ||
 683                             (fp_register_state_size == sizeof(struct x86_avx_thread_state)));
 684                         bcopy((char *)(ppcb->ifps),
 685                             (char *)(child->machine.ifps), fp_register_state_size);
 686
 687                         /* Mark the new fp saved state as non-live. */
 688                         /* Temporarily disabled: radar 4647827
 689                          * new_ifps->fp_valid = TRUE;
 690                          */
 691
 692                         /*
 693                          * Clear any reserved bits in the MXCSR to prevent a GPF
 694                          * when issuing an FXRSTOR.
 695                          */
 696                         new_ifps->fx_MXCSR &= mxcsr_capability_mask;
 697                         new_ifps = NULL;
 698                 }
 699         }
 700         simple_unlock(&ppcb->lock);
 701
 702         if (new_ifps != NULL)
 703                 fp_state_free(new_ifps);
 704 }
 705
 706
 707 /*
 708  * Initialize FPU.
 709  *
 710  */
 711
 712 void
 713 fpinit(void)
 714 {
 715         unsigned short  control;
 716
 717         clear_ts();
 718         fninit();
 719         fnstcw(&control);
 720         control &= ~(FPC_PC|FPC_RC); /* Clear precision & rounding control */
 721         control |= (FPC_PC_64 |         /* Set precision */
 722                         FPC_RC_RN |     /* round-to-nearest */
 723                         FPC_ZE |        /* Suppress zero-divide */
 724                         FPC_OE |        /*  and overflow */
 725                         FPC_UE |        /*  underflow */
 726                         FPC_IE |        /* Allow NaNQs and +-INF */
 727                         FPC_DE |        /* Allow denorms as operands  */
 728                         FPC_PE);        /* No trap for precision loss */
 729         fldcw(control);
 730
 731         /* Initialize SSE/SSE2 */
 732         __builtin_ia32_ldmxcsr(0x1f80);
 733 }
 734
 735 /*
 736  * Coprocessor not present.
 737  */
 738
 739 void
 740 fpnoextflt(void)
 741 {
 742         boolean_t       intr;
 743         thread_t        thr_act;
 744         pcb_t           pcb;
 745         struct x86_fx_thread_state *ifps = 0;
 746
 747         thr_act = current_thread();
 748         pcb = THREAD_TO_PCB(thr_act);
 749
 750         assert(fp_register_state_size != 0);
 751
 752         if (pcb->ifps == 0 && !get_interrupt_level()) {
 753                 ifps = fp_state_alloc();
 754                 bcopy((char *)&initial_fp_state, (char *)ifps,
 755                     fp_register_state_size);
 756                 if (!thread_is_64bit(thr_act)) {
 757                         ifps->fp_save_layout = fpu_YMM_present ? XSAVE32 : FXSAVE32;
 758                 }
 759                 else
 760                         ifps->fp_save_layout = fpu_YMM_present ? XSAVE64 : FXSAVE64;
 761                 ifps->fp_valid = TRUE;
 762         }
 763         intr = ml_set_interrupts_enabled(FALSE);
 764
 765         clear_ts();                     /*  Enable FPU use */
 766
 767         if (__improbable(get_interrupt_level())) {
 768                 /*
 769                  * Save current coprocessor context if valid
 770                  * Initialize coprocessor live context
 771                  */
 772                 fp_save(thr_act);
 773                 fpinit();
 774         } else {
 775                 if (pcb->ifps == 0) {
 776                         pcb->ifps = ifps;
 777                         ifps = 0;
 778                 }
 779                 /*
 780                  * Load this thread`s state into coprocessor live context.
 781                  */
 782                 fp_load(thr_act);
 783         }
 784         (void)ml_set_interrupts_enabled(intr);
 785
 786         if (ifps)
 787                 fp_state_free(ifps);
 788 }
 789
 790 /*
 791  * FPU overran end of segment.
 792  * Re-initialize FPU.  Floating point state is not valid.
 793  */
 794
 795 void
 796 fpextovrflt(void)
 797 {
 798         thread_t        thr_act = current_thread();
 799         pcb_t           pcb;
 800         struct x86_fx_thread_state *ifps;
 801         boolean_t       intr;
 802
 803         intr = ml_set_interrupts_enabled(FALSE);
 804
 805         if (get_interrupt_level())
 806                 panic("FPU segment overrun exception  at interrupt context\n");
 807         if (current_task() == kernel_task)
 808                 panic("FPU segment overrun exception in kernel thread context\n");
 809
 810         /*
 811          * This is a non-recoverable error.
 812          * Invalidate the thread`s FPU state.
 813          */
 814         pcb = THREAD_TO_PCB(thr_act);
 815         simple_lock(&pcb->lock);
 816         ifps = pcb->ifps;
 817         pcb->ifps = 0;
 818         simple_unlock(&pcb->lock);
 819
 820         /*
 821          * Re-initialize the FPU.
 822          */
 823         clear_ts();
 824         fninit();
 825
 826         /*
 827          * And disable access.
 828          */
 829         clear_fpu();
 830
 831         (void)ml_set_interrupts_enabled(intr);
 832
 833         if (ifps)
 834             zfree(ifps_zone, ifps);
 835
 836         /*
 837          * Raise exception.
 838          */
 839         i386_exception(EXC_BAD_ACCESS, VM_PROT_READ|VM_PROT_EXECUTE, 0);
 840         /*NOTREACHED*/
 841 }
 842
 843 /*
 844  * FPU error. Called by AST.
 845  */
 846
 847 void
 848 fpexterrflt(void)
 849 {
 850         thread_t        thr_act = current_thread();
 851         struct x86_fx_thread_state *ifps = thr_act->machine.ifps;
 852         boolean_t       intr;
 853
 854         intr = ml_set_interrupts_enabled(FALSE);
 855
 856         if (get_interrupt_level())
 857                 panic("FPU error exception at interrupt context\n");
 858         if (current_task() == kernel_task)
 859                 panic("FPU error exception in kernel thread context\n");
 860
 861         /*
 862          * Save the FPU state and turn off the FPU.
 863          */
 864         fp_save(thr_act);
 865
 866         (void)ml_set_interrupts_enabled(intr);
 867
 868         /*
 869          * Raise FPU exception.
 870          * Locking not needed on pcb->ifps,
 871          * since thread is running.
 872          */
 873         i386_exception(EXC_ARITHMETIC,
 874                        EXC_I386_EXTERR,
 875                        ifps->fx_status);
 876
 877         /*NOTREACHED*/
 878 }
 879
 880 /*
 881  * Save FPU state.
 882  *
 883  * Locking not needed:
 884  * .    if called from fpu_get_state, pcb already locked.
 885  * .    if called from fpnoextflt or fp_intr, we are single-cpu
 886  * .    otherwise, thread is running.
 887  * N.B.: Must be called with interrupts disabled
 888  */
 889
 890 void
 891 fp_save(
 892         thread_t        thr_act)
 893 {
 894         pcb_t pcb = THREAD_TO_PCB(thr_act);
 895         struct x86_fx_thread_state *ifps = pcb->ifps;
 896
 897         assert(ifps != 0);
 898         if (ifps != 0 && !ifps->fp_valid) {
 899                 assert((get_cr0() & CR0_TS) == 0);
 900                 /* registers are in FPU */
 901                 ifps->fp_valid = TRUE;
 902                 fpu_store_registers(ifps, thread_is_64bit(thr_act));
 903         }
 904 }
 905
 906 /*
 907  * Restore FPU state from PCB.
 908  *
 909  * Locking not needed; always called on the current thread.
 910  */
 911
 912 void
 913 fp_load(
 914         thread_t        thr_act)
 915 {
 916         pcb_t pcb = THREAD_TO_PCB(thr_act);
 917         struct x86_fx_thread_state *ifps = pcb->ifps;
 918
 919         assert(ifps);
 920 #if     DEBUG
 921         if (ifps->fp_valid != FALSE && ifps->fp_valid != TRUE) {
 922                 panic("fp_load() invalid fp_valid: %u, fp_save_layout: %u\n",
 923                       ifps->fp_valid, ifps->fp_save_layout);
 924         }
 925 #endif
 926
 927         if (ifps->fp_valid == FALSE) {
 928                 fpinit();
 929         } else {
 930                 fpu_load_registers(ifps);
 931         }
 932         ifps->fp_valid = FALSE;         /* in FPU */
 933 }
 934
 935 /*
 936  * SSE arithmetic exception handling code.
 937  * Basically the same as the x87 exception handler with a different subtype
 938  */
 939
 940 void
 941 fpSSEexterrflt(void)
 942 {
 943         thread_t        thr_act = current_thread();
 944         struct x86_fx_thread_state *ifps = thr_act->machine.ifps;
 945         boolean_t       intr;
 946
 947         intr = ml_set_interrupts_enabled(FALSE);
 948
 949         if (get_interrupt_level())
 950                 panic("SSE exception at interrupt context\n");
 951         if (current_task() == kernel_task)
 952                 panic("SSE exception in kernel thread context\n");
 953
 954         /*
 955          * Save the FPU state and turn off the FPU.
 956          */
 957         fp_save(thr_act);
 958
 959         (void)ml_set_interrupts_enabled(intr);
 960         /*
 961          * Raise FPU exception.
 962          * Locking not needed on pcb->ifps,
 963          * since thread is running.
 964          */
 965         assert(ifps->fp_save_layout == FXSAVE32 || ifps->fp_save_layout == FXSAVE64);
 966         i386_exception(EXC_ARITHMETIC,
 967                        EXC_I386_SSEEXTERR,
 968                        ifps->fx_MXCSR);
 969         /*NOTREACHED*/
 970 }
 971
 972 void
 973 fp_setvalid(boolean_t value) {
 974         thread_t        thr_act = current_thread();
 975         struct x86_fx_thread_state *ifps = thr_act->machine.ifps;
 976
 977         if (ifps) {
 978                 ifps->fp_valid = value;
 979
 980                 if (value == TRUE) {
 981                         boolean_t istate = ml_set_interrupts_enabled(FALSE);
 982                         clear_fpu();
 983                         ml_set_interrupts_enabled(istate);
 984                 }
 985         }
 986 }
 987
 988 boolean_t
 989 ml_fpu_avx_enabled(void) {
 990         return (fpu_YMM_present == TRUE);
 991 }