xnu-4570.41.2.tar.gz

[apple/xnu.git] / osfmk / i386 / pcb.c

diff --git a/osfmk/i386/pcb.c b/osfmk/i386/pcb.c
index cc5b22787c3d70ac32cd066f84587aeb0d369ccf..5fb9112ea51416f0dadb13a39bcbe4782a9e2a1f 100644 (file)
--- a/osfmk/i386/pcb.c
+++ b/osfmk/i386/pcb.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2016 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -54,7 +54,6 @@
  * the rights to redistribute these changes.
  */
 
  * the rights to redistribute these changes.
  */
 

-#include <mach_rt.h>

 #include <mach_debug.h>
 #include <mach_ldebug.h>
 
 #include <mach_debug.h>
 #include <mach_ldebug.h>
 


@@ -77,6 +76,7 @@
 #include <kern/assert.h>
 #include <kern/spl.h>
 #include <kern/machine.h>
 #include <kern/assert.h>
 #include <kern/spl.h>
 #include <kern/machine.h>

+#include <kern/kpc.h>

 #include <ipc/ipc_port.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_map.h>
 #include <ipc/ipc_port.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_map.h>


@@ -94,39 +94,34 @@
 #include <i386/machine_routines.h>
 #include <i386/lapic.h> /* LAPIC_PMC_SWI_VECTOR */
 
 #include <i386/machine_routines.h>
 #include <i386/lapic.h> /* LAPIC_PMC_SWI_VECTOR */
 

-#if CONFIG_COUNTERS
-#include <pmc/pmc.h>
-#endif /* CONFIG_COUNTERS */
-
-#if KPC
-#include <kern/kpc.h>
-#endif
-
-#if KPERF
-#include <kperf/kperf.h>
+#if HYPERVISOR
+#include <kern/hv_support.h>

 #endif
 
 /*
  * Maps state flavor to number of words in the state:
  */
 unsigned int _MachineStateCount[] = {
 #endif
 
 /*
  * Maps state flavor to number of words in the state:
  */
 unsigned int _MachineStateCount[] = {

-       /* FLAVOR_LIST */
-        0,
-       x86_THREAD_STATE32_COUNT,
-       x86_FLOAT_STATE32_COUNT,
-       x86_EXCEPTION_STATE32_COUNT,
-       x86_THREAD_STATE64_COUNT,
-       x86_FLOAT_STATE64_COUNT,
-       x86_EXCEPTION_STATE64_COUNT,
-       x86_THREAD_STATE_COUNT,
-       x86_FLOAT_STATE_COUNT,
-       x86_EXCEPTION_STATE_COUNT,
-       0,
-       x86_SAVED_STATE32_COUNT,
-       x86_SAVED_STATE64_COUNT,
-       x86_DEBUG_STATE32_COUNT,
-       x86_DEBUG_STATE64_COUNT,
-       x86_DEBUG_STATE_COUNT
+       [x86_THREAD_STATE32]    = x86_THREAD_STATE32_COUNT,
+       [x86_THREAD_STATE64]    = x86_THREAD_STATE64_COUNT,
+       [x86_THREAD_STATE]      = x86_THREAD_STATE_COUNT,
+       [x86_FLOAT_STATE32]     = x86_FLOAT_STATE32_COUNT,
+       [x86_FLOAT_STATE64]     = x86_FLOAT_STATE64_COUNT,
+       [x86_FLOAT_STATE]       = x86_FLOAT_STATE_COUNT,
+       [x86_EXCEPTION_STATE32] = x86_EXCEPTION_STATE32_COUNT,
+       [x86_EXCEPTION_STATE64] = x86_EXCEPTION_STATE64_COUNT,
+       [x86_EXCEPTION_STATE]   = x86_EXCEPTION_STATE_COUNT,
+       [x86_DEBUG_STATE32]     = x86_DEBUG_STATE32_COUNT,
+       [x86_DEBUG_STATE64]     = x86_DEBUG_STATE64_COUNT,
+       [x86_DEBUG_STATE]       = x86_DEBUG_STATE_COUNT,
+       [x86_AVX_STATE32]       = x86_AVX_STATE32_COUNT,
+       [x86_AVX_STATE64]       = x86_AVX_STATE64_COUNT,
+       [x86_AVX_STATE]         = x86_AVX_STATE_COUNT,
+#if !defined(RC_HIDE_XNU_J137)
+       [x86_AVX512_STATE32]    = x86_AVX512_STATE32_COUNT,
+       [x86_AVX512_STATE64]    = x86_AVX512_STATE64_COUNT,
+       [x86_AVX512_STATE]      = x86_AVX512_STATE_COUNT,
+#endif /* not RC_HIDE_XNU_J137 */

 };
 
 zone_t         iss_zone;               /* zone for saved_state area */
 };
 
 zone_t         iss_zone;               /* zone for saved_state area */


@@ -136,7 +131,7 @@ zone_t              ids_zone;               /* zone for debug_state area */
 
 extern void            Thread_continue(void);
 extern void            Load_context(
 
 extern void            Thread_continue(void);
 extern void            Load_context(

-                               thread_t                        thread);
+                               thread_t                        thread) __attribute__((noreturn));

 
 static void
 get_exception_state32(thread_t thread, x86_exception_state32_t *es);
 
 static void
 get_exception_state32(thread_t thread, x86_exception_state32_t *es);


@@ -156,66 +151,26 @@ set_thread_state32(thread_t thread, x86_thread_state32_t *ts);
 static int
 set_thread_state64(thread_t thread, x86_thread_state64_t *ts);
 
 static int
 set_thread_state64(thread_t thread, x86_thread_state64_t *ts);
 

-#if CONFIG_COUNTERS
-static inline void
-machine_pmc_cswitch(thread_t /* old */, thread_t /* new */);
-
-static inline void
-pmc_swi(thread_t /* old */, thread_t /*new */);
-
-static inline void
-pmc_swi(thread_t old, thread_t new) {
-       current_cpu_datap()->csw_old_thread = old;
-       current_cpu_datap()->csw_new_thread = new;
-       pal_pmc_swi();
-}
-
-static inline void
-machine_pmc_cswitch(thread_t old, thread_t new) {
-       if (pmc_thread_eligible(old) || pmc_thread_eligible(new)) {
-               pmc_swi(old, new);
-       }
-}
-
-void ml_get_csw_threads(thread_t *old, thread_t *new) {
-       *old = current_cpu_datap()->csw_old_thread;
-       *new = current_cpu_datap()->csw_new_thread;
-}
-
-#endif /* CONFIG_COUNTERS */
-
-#if KPC
+#if HYPERVISOR

 static inline void
 static inline void

-ml_kpc_cswitch(thread_t old, thread_t new)
+ml_hv_cswitch(thread_t old, thread_t new)

 {
 {

-       if(!kpc_threads_counting)
-               return;
-       
-       /* call the kpc function */
-       kpc_switch_context( old, new );
-}
-#endif
+       if (old->hv_thread_target)
+               hv_callbacks.preempt(old->hv_thread_target);

 
 

-#if KPERF
-static inline void
-ml_kperf_cswitch(thread_t old, thread_t new)
-{
-       if(!kperf_cswitch_hook)
-               return;
-       
-       /* call the kpc function */
-       kperf_switch_context( old, new );
+       if (new->hv_thread_target)
+               hv_callbacks.dispatch(new->hv_thread_target);   

 }
 #endif
 
 /*
 }
 #endif
 
 /*

- * Don't let an illegal value for dr7 get set. Specifically,
- * check for undefined settings.  Setting these bit patterns
+ * Don't let an illegal value for the lower 32-bits of dr7 get set.
+ * Specifically, check for undefined settings.  Setting these bit patterns

  * result in undefined behaviour and can lead to an unexpected
  * TRCTRAP.
  */
 static boolean_t
  * result in undefined behaviour and can lead to an unexpected
  * TRCTRAP.
  */
 static boolean_t

-dr7_is_valid(uint32_t *dr7)
+dr7d_is_valid(uint32_t *dr7d)

 {
        int i;
        uint32_t mask1, mask2;
 {
        int i;
        uint32_t mask1, mask2;


@@ -227,7 +182,7 @@ dr7_is_valid(uint32_t *dr7)
        if (!(get_cr4() & CR4_DE))
                for (i = 0, mask1 = 0x3<<16, mask2 = 0x2<<16; i < 4; 
                                i++, mask1 <<= 4, mask2 <<= 4)
        if (!(get_cr4() & CR4_DE))
                for (i = 0, mask1 = 0x3<<16, mask2 = 0x2<<16; i < 4; 
                                i++, mask1 <<= 4, mask2 <<= 4)

-                       if ((*dr7 & mask1) == mask2)
+                       if ((*dr7d & mask1) == mask2)

                                return (FALSE);
 
        /*
                                return (FALSE);
 
        /*


@@ -236,75 +191,53 @@ dr7_is_valid(uint32_t *dr7)
         * to "00B"
         */
        for (i = 0; i < 4; i++)
         * to "00B"
         */
        for (i = 0; i < 4; i++)

-               if (((((*dr7 >> (16 + i*4))) & 0x3) == 0) &&
-                               ((((*dr7 >> (18 + i*4))) & 0x3) != 0))
+               if (((((*dr7d >> (16 + i*4))) & 0x3) == 0) &&
+                               ((((*dr7d >> (18 + i*4))) & 0x3) != 0))

                        return (FALSE);
 
        /*
         * Intel docs have these bits fixed.
         */
                        return (FALSE);
 
        /*
         * Intel docs have these bits fixed.
         */

-       *dr7 |= 0x1 << 10; /* set bit 10 to 1 */
-       *dr7 &= ~(0x1 << 11); /* set bit 11 to 0 */
-       *dr7 &= ~(0x1 << 12); /* set bit 12 to 0 */
-       *dr7 &= ~(0x1 << 14); /* set bit 14 to 0 */
-       *dr7 &= ~(0x1 << 15); /* set bit 15 to 0 */
+       *dr7d |= 0x1 << 10; /* set bit 10 to 1 */
+       *dr7d &= ~(0x1 << 11); /* set bit 11 to 0 */
+       *dr7d &= ~(0x1 << 12); /* set bit 12 to 0 */
+       *dr7d &= ~(0x1 << 14); /* set bit 14 to 0 */
+       *dr7d &= ~(0x1 << 15); /* set bit 15 to 0 */

 
        /*
         * We don't allow anything to set the global breakpoints.
         */
 
 
        /*
         * We don't allow anything to set the global breakpoints.
         */
 

-       if (*dr7 & 0x2)
+       if (*dr7d & 0x2)

                return (FALSE);
 
                return (FALSE);
 

-       if (*dr7 & (0x2<<2))
+       if (*dr7d & (0x2<<2))

                return (FALSE);
 
                return (FALSE);
 

-       if (*dr7 & (0x2<<4))
+       if (*dr7d & (0x2<<4))

                return (FALSE);
 
                return (FALSE);
 

-       if (*dr7 & (0x2<<6))
+       if (*dr7d & (0x2<<6))

                return (FALSE);
 
        return (TRUE);
 }
 
                return (FALSE);
 
        return (TRUE);
 }
 

-static inline void
-set_live_debug_state32(cpu_data_t *cdp, x86_debug_state32_t *ds)
-{
-       __asm__ volatile ("movl %0,%%db0" : :"r" (ds->dr0));
-       __asm__ volatile ("movl %0,%%db1" : :"r" (ds->dr1));
-       __asm__ volatile ("movl %0,%%db2" : :"r" (ds->dr2));
-       __asm__ volatile ("movl %0,%%db3" : :"r" (ds->dr3));
-       cdp->cpu_dr7 = ds->dr7;
-}
-

 extern void set_64bit_debug_regs(x86_debug_state64_t *ds);
 
 extern void set_64bit_debug_regs(x86_debug_state64_t *ds);
 

-static inline void
-set_live_debug_state64(cpu_data_t *cdp, x86_debug_state64_t *ds)
-{
-       /*
-        * We need to enter 64-bit mode in order to set the full
-        * width of these registers
-        */
-       set_64bit_debug_regs(ds);
-       cdp->cpu_dr7 = ds->dr7;
-}
-

 boolean_t
 debug_state_is_valid32(x86_debug_state32_t *ds) 
 {
 boolean_t
 debug_state_is_valid32(x86_debug_state32_t *ds) 
 {

-       if (!dr7_is_valid(&ds->dr7))
+       if (!dr7d_is_valid(&ds->dr7))

                return FALSE;
 
                return FALSE;
 

-

        return TRUE;
 }
 
 boolean_t
 debug_state_is_valid64(x86_debug_state64_t *ds)
 {
        return TRUE;
 }
 
 boolean_t
 debug_state_is_valid64(x86_debug_state64_t *ds)
 {

-       if (!dr7_is_valid((uint32_t *)&ds->dr7))
+       if (!dr7d_is_valid((uint32_t *)&ds->dr7))

                return FALSE;
 
        /*
                return FALSE;
 
        /*


@@ -327,6 +260,9 @@ debug_state_is_valid64(x86_debug_state64_t *ds)
                if (ds->dr3 >= VM_MAX_PAGE_ADDRESS)
                        return FALSE;
 
                if (ds->dr3 >= VM_MAX_PAGE_ADDRESS)
                        return FALSE;
 

+       /* For x86-64, we must ensure the upper 32-bits of DR7 are clear */
+       ds->dr7 &= 0xffffffffULL;
+

        return TRUE;
 }
 
        return TRUE;
 }
 


@@ -382,6 +318,13 @@ set_debug_state64(thread_t thread, x86_debug_state64_t *ds)
                ids = zalloc(ids_zone);
                bzero(ids, sizeof *ids);
 
                ids = zalloc(ids_zone);
                bzero(ids, sizeof *ids);
 

+#if HYPERVISOR
+               if (thread->hv_thread_target) {
+                       hv_callbacks.volatile_state(thread->hv_thread_target,
+                               HV_DEBUG_STATE);
+               }
+#endif
+

                simple_lock(&pcb->lock);
                /* make sure it wasn't already alloc()'d elsewhere */
                if (pcb->ids == NULL) {
                simple_lock(&pcb->lock);
                /* make sure it wasn't already alloc()'d elsewhere */
                if (pcb->ids == NULL) {


@@ -446,14 +389,20 @@ void
 machine_load_context(
        thread_t                new)
 {
 machine_load_context(
        thread_t                new)
 {

-#if CONFIG_COUNTERS
-       machine_pmc_cswitch(NULL, new);
-#endif

        new->machine.specFlags |= OnProc;
        act_machine_switch_pcb(NULL, new);
        Load_context(new);
 }
 
        new->machine.specFlags |= OnProc;
        act_machine_switch_pcb(NULL, new);
        Load_context(new);
 }
 

+static inline void pmap_switch_context(thread_t ot, thread_t nt, int cnum) {
+       pmap_assert(ml_get_interrupts_enabled() == FALSE);
+       vm_map_t nmap = nt->map, omap = ot->map;
+       if ((omap != nmap) || (nmap->pmap->pagezero_accessible)) {
+               PMAP_DEACTIVATE_MAP(omap, ot, cnum);
+               PMAP_ACTIVATE_MAP(nmap, nt, cnum);
+       }
+}
+

 /*
  * Switch to a new thread.
  * Save the old thread`s kernel state or continuation,
 /*
  * Switch to a new thread.
  * Save the old thread`s kernel state or continuation,


@@ -465,28 +414,22 @@ machine_switch_context(
        thread_continue_t       continuation,
        thread_t                        new)
 {
        thread_continue_t       continuation,
        thread_t                        new)
 {

-#if MACH_RT
-        assert(current_cpu_datap()->cpu_active_stack == old->kernel_stack);
-#endif
-#if CONFIG_COUNTERS
-       machine_pmc_cswitch(old, new);
-#endif
+       assert(current_cpu_datap()->cpu_active_stack == old->kernel_stack);
+

 #if KPC
 #if KPC

-       ml_kpc_cswitch(old, new);
-#endif
-#if KPERF
-       ml_kperf_cswitch(old, new);
-#endif
+       kpc_off_cpu(old);
+#endif /* KPC */
+

        /*
         *      Save FP registers if in use.
         */
        /*
         *      Save FP registers if in use.
         */

-       fpu_save_context(old);
+       fpu_switch_context(old, new);

 
        old->machine.specFlags &= ~OnProc;
        new->machine.specFlags |= OnProc;
 
        /*
 
        old->machine.specFlags &= ~OnProc;
        new->machine.specFlags |= OnProc;
 
        /*

-        * Monitor the stack depth and report new max,
+        * Monitor the stack depth and report new max,

         * not worrying about races.
         */
        vm_offset_t     depth = current_stack_depth();
         * not worrying about races.
         */
        vm_offset_t     depth = current_stack_depth();


@@ -501,13 +444,17 @@ machine_switch_context(
         *      Switch address maps if need be, even if not switching tasks.
         *      (A server activation may be "borrowing" a client map.)
         */
         *      Switch address maps if need be, even if not switching tasks.
         *      (A server activation may be "borrowing" a client map.)
         */

-       PMAP_SWITCH_CONTEXT(old, new, cpu_number());
+       pmap_switch_context(old, new, cpu_number());

 
        /*
         *      Load the rest of the user state for the new thread
         */
        act_machine_switch_pcb(old, new);
 
 
        /*
         *      Load the rest of the user state for the new thread
         */
        act_machine_switch_pcb(old, new);
 

+#if HYPERVISOR
+       ml_hv_cswitch(old, new);
+#endif
+

        return(Switch_context(old, continuation, new));
 }
 
        return(Switch_context(old, continuation, new));
 }
 


@@ -520,8 +467,8 @@ machine_processor_shutdown(
 #if CONFIG_VMX
        vmx_suspend();
 #endif
 #if CONFIG_VMX
        vmx_suspend();
 #endif

-       fpu_save_context(thread);
-       PMAP_SWITCH_CONTEXT(thread, processor->idle_thread, cpu_number());
+       fpu_switch_context(thread, NULL);
+       pmap_switch_context(thread, processor->idle_thread, cpu_number());

        return(Shutdown_context(thread, doshutdown, processor));
 }
 
        return(Shutdown_context(thread, doshutdown, processor));
 }
 


@@ -636,6 +583,9 @@ set_thread_state32(thread_t thread, x86_thread_state32_t *ts)
        ts->ds = USER_DS;
        ts->es = USER_DS;
 
        ts->ds = USER_DS;
        ts->es = USER_DS;
 

+       /* Set GS to CTHREAD only if's been established */
+       ts->gs = thread->machine.cthread_self ? USER_CTHREAD : NULL_SEG;
+ 

        /* Check segment selectors are safe */
        if (!valid_user_segment_selectors(ts->cs,
                                          ts->ss,
        /* Check segment selectors are safe */
        if (!valid_user_segment_selectors(ts->cs,
                                          ts->ss,


@@ -923,8 +873,12 @@ machine_thread_set_state(
        }
 
        case x86_FLOAT_STATE32:
        }
 
        case x86_FLOAT_STATE32:

+       case x86_AVX_STATE32:
+#if !defined(RC_HIDE_XNU_J137)
+       case x86_AVX512_STATE32:
+#endif /* not RC_HIDE_XNU_J137 */

        {
        {

-               if (count != x86_FLOAT_STATE32_COUNT)
+               if (count != _MachineStateCount[flavor])

                        return(KERN_INVALID_ARGUMENT);
 
                if (thread_is_64bit(thr_act))
                        return(KERN_INVALID_ARGUMENT);
 
                if (thread_is_64bit(thr_act))


@@ -934,11 +888,15 @@ machine_thread_set_state(
        }
 
        case x86_FLOAT_STATE64:
        }
 
        case x86_FLOAT_STATE64:

+       case x86_AVX_STATE64:
+#if !defined(RC_HIDE_XNU_J137)
+       case x86_AVX512_STATE64:
+#endif /* not RC_HIDE_XNU_J137 */

        {
        {

-               if (count != x86_FLOAT_STATE64_COUNT)
+               if (count != _MachineStateCount[flavor])

                        return(KERN_INVALID_ARGUMENT);
 
                        return(KERN_INVALID_ARGUMENT);
 

-               if ( !thread_is_64bit(thr_act))
+               if (!thread_is_64bit(thr_act))

                        return(KERN_INVALID_ARGUMENT);
 
                return fpu_set_fxstate(thr_act, tstate, flavor);
                        return(KERN_INVALID_ARGUMENT);
 
                return fpu_set_fxstate(thr_act, tstate, flavor);


@@ -963,49 +921,33 @@ machine_thread_set_state(
                return(KERN_INVALID_ARGUMENT);
        }
 
                return(KERN_INVALID_ARGUMENT);
        }
 

-       case x86_AVX_STATE32:
-       {
-               if (count != x86_AVX_STATE32_COUNT)
-                       return(KERN_INVALID_ARGUMENT);
-
-               if (thread_is_64bit(thr_act))
-                       return(KERN_INVALID_ARGUMENT);
-
-               return fpu_set_fxstate(thr_act, tstate, flavor);
-       }
-
-       case x86_AVX_STATE64:
-       {
-               if (count != x86_AVX_STATE64_COUNT)
-                       return(KERN_INVALID_ARGUMENT);
-
-               if (!thread_is_64bit(thr_act))
-                       return(KERN_INVALID_ARGUMENT);
-
-               return fpu_set_fxstate(thr_act, tstate, flavor);
-       }
-

        case x86_AVX_STATE:
        case x86_AVX_STATE:

+#if !defined(RC_HIDE_XNU_J137)
+       case x86_AVX512_STATE:
+#endif

        {   
                x86_avx_state_t       *state;
 
        {   
                x86_avx_state_t       *state;
 

-               if (count != x86_AVX_STATE_COUNT)
+               if (count != _MachineStateCount[flavor])

                        return(KERN_INVALID_ARGUMENT);
 
                state = (x86_avx_state_t *)tstate;
                        return(KERN_INVALID_ARGUMENT);
 
                state = (x86_avx_state_t *)tstate;

-               if (state->ash.flavor == x86_AVX_STATE64 &&
-                   state->ash.count  == x86_FLOAT_STATE64_COUNT &&
+               /* Flavors are defined to have sequential values: 32-bit, 64-bit, non-specific */
+               /* 64-bit flavor? */
+               if (state->ash.flavor == (flavor - 1) &&
+                   state->ash.count  == _MachineStateCount[flavor - 1] &&

                    thread_is_64bit(thr_act)) {
                        return fpu_set_fxstate(thr_act,
                                               (thread_state_t)&state->ufs.as64,
                    thread_is_64bit(thr_act)) {
                        return fpu_set_fxstate(thr_act,
                                               (thread_state_t)&state->ufs.as64,

-                                              x86_FLOAT_STATE64);
+                                              flavor - 1);

                }
                }

-               if (state->ash.flavor == x86_FLOAT_STATE32 &&
-                   state->ash.count  == x86_FLOAT_STATE32_COUNT &&
+               /* 32-bit flavor? */
+               if (state->ash.flavor == (flavor - 2) &&
+                   state->ash.count  == _MachineStateCount[flavor - 2] &&

                    !thread_is_64bit(thr_act)) {
                        return fpu_set_fxstate(thr_act,
                                               (thread_state_t)&state->ufs.as32,
                    !thread_is_64bit(thr_act)) {
                        return fpu_set_fxstate(thr_act,
                                               (thread_state_t)&state->ufs.as32,

-                                              x86_FLOAT_STATE32); 
+                                              flavor - 2); 

                }
                return(KERN_INVALID_ARGUMENT);
        }
                }
                return(KERN_INVALID_ARGUMENT);
        }


@@ -1051,8 +993,6 @@ machine_thread_set_state(
                        return set_thread_state32(thr_act, &state->uts.ts32);
                } else
                        return(KERN_INVALID_ARGUMENT);
                        return set_thread_state32(thr_act, &state->uts.ts32);
                } else
                        return(KERN_INVALID_ARGUMENT);

-
-               break;

        }
        case x86_DEBUG_STATE32:
        {
        }
        case x86_DEBUG_STATE32:
        {


@@ -1171,6 +1111,24 @@ machine_thread_get_state(
                break;
            }
 
                break;
            }
 

+#if !defined(RC_HIDE_XNU_J137)
+           case THREAD_STATE_FLAVOR_LIST_10_13:
+           {
+               if (*count < 6)
+                       return (KERN_INVALID_ARGUMENT);
+
+               tstate[0] = x86_THREAD_STATE;
+               tstate[1] = x86_FLOAT_STATE;
+               tstate[2] = x86_EXCEPTION_STATE;
+               tstate[3] = x86_DEBUG_STATE;
+               tstate[4] = x86_AVX_STATE;
+               tstate[5] = x86_AVX512_STATE;
+
+               *count = 6;
+               break;
+           }
+
+#endif

            case x86_SAVED_STATE32:
            {
                x86_saved_state32_t     *state;
            case x86_SAVED_STATE32:
            {
                x86_saved_state32_t     *state;


@@ -1280,58 +1238,64 @@ machine_thread_get_state(
            }
 
            case x86_AVX_STATE32:
            }
 
            case x86_AVX_STATE32:

+#if !defined(RC_HIDE_XNU_J137)
+           case x86_AVX512_STATE32:
+#endif

            {
            {

-               if (*count != x86_AVX_STATE32_COUNT)
+               if (*count != _MachineStateCount[flavor])

                        return(KERN_INVALID_ARGUMENT);
 
                if (thread_is_64bit(thr_act))
                        return(KERN_INVALID_ARGUMENT);
 
                        return(KERN_INVALID_ARGUMENT);
 
                if (thread_is_64bit(thr_act))
                        return(KERN_INVALID_ARGUMENT);
 

-               *count = x86_AVX_STATE32_COUNT;
+               *count = _MachineStateCount[flavor];

 
                return fpu_get_fxstate(thr_act, tstate, flavor);
            }
 
            case x86_AVX_STATE64:
 
                return fpu_get_fxstate(thr_act, tstate, flavor);
            }
 
            case x86_AVX_STATE64:

+#if !defined(RC_HIDE_XNU_J137)
+           case x86_AVX512_STATE64:
+#endif

            {
            {

-               if (*count != x86_AVX_STATE64_COUNT)
+               if (*count != _MachineStateCount[flavor])

                        return(KERN_INVALID_ARGUMENT);
 
                if ( !thread_is_64bit(thr_act))
                        return(KERN_INVALID_ARGUMENT);
 
                        return(KERN_INVALID_ARGUMENT);
 
                if ( !thread_is_64bit(thr_act))
                        return(KERN_INVALID_ARGUMENT);
 

-               *count = x86_AVX_STATE64_COUNT;
+               *count = _MachineStateCount[flavor];

 
                return fpu_get_fxstate(thr_act, tstate, flavor);
            }
 
            case x86_AVX_STATE:
 
                return fpu_get_fxstate(thr_act, tstate, flavor);
            }
 
            case x86_AVX_STATE:

+#if !defined(RC_HIDE_XNU_J137)
+           case x86_AVX512_STATE:
+#endif

            {
                x86_avx_state_t         *state;
            {
                x86_avx_state_t         *state;

-               kern_return_t           kret;
+               thread_state_t          fstate;

 
 

-               if (*count < x86_AVX_STATE_COUNT)
+               if (*count < _MachineStateCount[flavor])

                        return(KERN_INVALID_ARGUMENT);
 
                        return(KERN_INVALID_ARGUMENT);
 

+               *count = _MachineStateCount[flavor];

                state = (x86_avx_state_t *)tstate;
 
                state = (x86_avx_state_t *)tstate;
 

-               bzero((char *)state, sizeof(x86_avx_state_t));
+               bzero((char *)state, *count * sizeof(int));
+

                if (thread_is_64bit(thr_act)) {
                if (thread_is_64bit(thr_act)) {

-                       state->ash.flavor = x86_AVX_STATE64;
-                       state->ash.count  = x86_AVX_STATE64_COUNT;
-                       kret = fpu_get_fxstate(thr_act,
-                                              (thread_state_t)&state->ufs.as64,
-                                              x86_AVX_STATE64);
+                       flavor -= 1;    /* 64-bit flavor */
+                       fstate = (thread_state_t) &state->ufs.as64;

                } else {
                } else {

-                       state->ash.flavor = x86_AVX_STATE32;
-                       state->ash.count  = x86_AVX_STATE32_COUNT;
-                       kret = fpu_get_fxstate(thr_act,
-                                              (thread_state_t)&state->ufs.as32,
-                                              x86_AVX_STATE32);
+                       flavor -= 2;    /* 32-bit flavor */
+                       fstate = (thread_state_t) &state->ufs.as32;

                }
                }

-               *count = x86_AVX_STATE_COUNT;
+               state->ash.flavor = flavor; 
+               state->ash.count  = _MachineStateCount[flavor];

 
 

-               return(kret);
+               return fpu_get_fxstate(thr_act, fstate, flavor);

            }
 
            case x86_THREAD_STATE32: 
            }
 
            case x86_THREAD_STATE32: 


@@ -1698,6 +1662,9 @@ machine_thread_switch_addrmode(thread_t thread)
         */
        machine_thread_create(thread, thread->task);
 
         */
        machine_thread_create(thread, thread->task);
 

+       /* Adjust FPU state */
+       fpu_switch_addrmode(thread, task_has_64BitAddr(thread->task));
+

        /* If we're switching ourselves, reset the pcb addresses etc. */
        if (thread == current_thread()) {
                boolean_t istate = ml_set_interrupts_enabled(FALSE);
        /* If we're switching ourselves, reset the pcb addresses etc. */
        if (thread == current_thread()) {
                boolean_t istate = ml_set_interrupts_enabled(FALSE);


@@ -1798,16 +1765,17 @@ machine_stack_attach(
 
        assert(stack);
        thread->kernel_stack = stack;
 
        assert(stack);
        thread->kernel_stack = stack;

+       thread_initialize_kernel_state(thread);

 
        statep = STACK_IKS(stack);
 #if defined(__x86_64__)
        statep->k_rip = (unsigned long) Thread_continue;
        statep->k_rbx = (unsigned long) thread_continue;
 
        statep = STACK_IKS(stack);
 #if defined(__x86_64__)
        statep->k_rip = (unsigned long) Thread_continue;
        statep->k_rbx = (unsigned long) thread_continue;

-       statep->k_rsp = (unsigned long) (STACK_IKS(stack) - 1);
+       statep->k_rsp = (unsigned long) STACK_IKS(stack);

 #else
        statep->k_eip = (unsigned long) Thread_continue;
        statep->k_ebx = (unsigned long) thread_continue;
 #else
        statep->k_eip = (unsigned long) Thread_continue;
        statep->k_ebx = (unsigned long) thread_continue;

-       statep->k_esp = (unsigned long) (STACK_IKS(stack) - 1);
+       statep->k_esp = (unsigned long) STACK_IKS(stack);

 #endif
 
        return;
 #endif
 
        return;


@@ -1826,15 +1794,7 @@ machine_stack_handoff(thread_t old,
        assert(new);
        assert(old);
 
        assert(new);
        assert(old);
 

-#if CONFIG_COUNTERS
-       machine_pmc_cswitch(old, new);
-#endif
-#if KPC
-       ml_kpc_cswitch(old, new);
-#endif
-#if KPERF
-       ml_kperf_cswitch(old, new);
-#endif
+       kpc_off_cpu(old);

 
        stack = old->kernel_stack;
        if (stack == old->reserved_stack) {
 
        stack = old->kernel_stack;
        if (stack == old->reserved_stack) {


@@ -1849,15 +1809,20 @@ machine_stack_handoff(thread_t old,
         */
        new->kernel_stack = stack;
 
         */
        new->kernel_stack = stack;
 

-       fpu_save_context(old);
+       fpu_switch_context(old, new);

        
        old->machine.specFlags &= ~OnProc;
        new->machine.specFlags |= OnProc;
 
        
        old->machine.specFlags &= ~OnProc;
        new->machine.specFlags |= OnProc;
 

-       PMAP_SWITCH_CONTEXT(old, new, cpu_number());
+       pmap_switch_context(old, new, cpu_number());

        act_machine_switch_pcb(old, new);
 
        act_machine_switch_pcb(old, new);
 

+#if HYPERVISOR
+       ml_hv_cswitch(old, new);
+#endif
+

        machine_set_current_thread(new);
        machine_set_current_thread(new);

+       thread_initialize_kernel_state(new);

 
        return;
 }
 
        return;
 }