xnu-6153.141.1.tar.gz

[apple/xnu.git] / osfmk / x86_64 / machine_routines_asm.s

diff --git a/osfmk/x86_64/machine_routines_asm.s b/osfmk/x86_64/machine_routines_asm.s
index 1c74f9fc8b3ed40d2551799ab5f9c6cce492c861..c0a1bdc90cde088a4d5a3440406f126ec61ed2ca 100644 (file)
--- a/osfmk/x86_64/machine_routines_asm.s
+++ b/osfmk/x86_64/machine_routines_asm.s
@@ -33,14 +33,13 @@
        
 #include <i386/postcode.h>
 #include <i386/apic.h>
        
 #include <i386/postcode.h>
 #include <i386/apic.h>

+#include <i386/vmx/vmx_asm.h>

 #include <assym.s>
 
 /*
 **      ml_get_timebase()
 **
 #include <assym.s>
 
 /*
 **      ml_get_timebase()
 **

-**      Entry   - %rdi contains pointer to 64 bit structure.
-**
-**      Exit    - 64 bit structure filled in.
+**      Returns TSC in RAX

 **
 */
 ENTRY(ml_get_timebase)
 **
 */
 ENTRY(ml_get_timebase)


@@ -50,7 +49,6 @@ ENTRY(ml_get_timebase)
        lfence
         shlq   $32,%rdx 
         orq    %rdx,%rax
        lfence
         shlq   $32,%rdx 
         orq    %rdx,%rax

-       movq    %rax, (%rdi)

                        
        ret
 
                        
        ret
 


@@ -80,10 +78,15 @@ ENTRY(ml_get_timebase)
  *
  */
 ENTRY(tmrCvt)
  *
  */
 ENTRY(tmrCvt)

+       cmpq    $1,%rsi                         /* check for unity fastpath */
+       je      1f

        movq    %rdi,%rax
        mulq    %rsi                            /* result is %rdx:%rax */
        shrdq   $32,%rdx,%rax                   /* %rdx:%rax >>= 32 */
        ret
        movq    %rdi,%rax
        mulq    %rsi                            /* result is %rdx:%rax */
        shrdq   $32,%rdx,%rax                   /* %rdx:%rax >>= 32 */
        ret

+1:
+       mov     %rdi,%rax
+       ret

 
  /*
  * void _rtc_nanotime_adjust(
 
  /*
  * void _rtc_nanotime_adjust(


@@ -103,28 +106,36 @@ ENTRY(_rtc_nanotime_adjust)
        ret
 
 /*
        ret
 
 /*

- * unint64_t _rtc_nanotime_read(rtc_nanotime_t *rntp, int slow);
+ * uint64_t _rtc_nanotime_read(rtc_nanotime_t *rntp);

  *
  * This is the same as the commpage nanotime routine, except that it uses the
  * kernel internal "rtc_nanotime_info" data instead of the commpage data.
  * These two copies of data are kept in sync by rtc_clock_napped().
  *
  *
  * This is the same as the commpage nanotime routine, except that it uses the
  * kernel internal "rtc_nanotime_info" data instead of the commpage data.
  * These two copies of data are kept in sync by rtc_clock_napped().
  *

- * Warning!  There is another copy of this code in osfmk/x86_64/idt64.s.
- * These are kept in sync by both using the RTC_NANOTIME_READ() macro.
+ * Warning!  There are several copies of this code in the trampolines found in
+ * osfmk/x86_64/idt64.s, coming from the various TIMER macros in rtclock_asm.h.
+ * They're all kept in sync by using the RTC_NANOTIME_READ() macro.
+ *
+ * The algorithm we use is:
+ *
+ *     ns = ((((rdtsc - rnt_tsc_base)<<rnt_shift)*rnt_tsc_scale) / 2**32) + rnt_ns_base;
+ *
+ * rnt_shift, a constant computed during initialization, is the smallest value for which:

  *
  *

- * There are two versions of this algorithm, for "slow" and "fast" processors.
- * The more common "fast" algorithm is:
+ *     (tscFreq << rnt_shift) > SLOW_TSC_THRESHOLD

  *
  *

- *     ns = (((rdtsc - rnt_tsc_base)*rnt_tsc_scale) / 2**32) + rnt_ns_base;
+ * Where SLOW_TSC_THRESHOLD is about 10e9.  Since most processor's tscFreqs are greater
+ * than 1GHz, rnt_shift is usually 0.  rnt_tsc_scale is also a 32-bit constant:

  *
  *

- * Of course, the divide by 2**32 is a nop.  rnt_tsc_scale is a constant
- * computed during initialization:
+ *     rnt_tsc_scale = (10e9 * 2**32) / (tscFreq << rnt_shift);

  *
  *

- *     rnt_tsc_scale = (10e9 * 2**32) / tscFreq;
+ * On 64-bit processors this algorithm could be simplified by doing a 64x64 bit
+ * multiply of rdtsc by tscFCvtt2n:

  *
  *

- * The "slow" algorithm uses long division:
+ *     ns = (((rdtsc - rnt_tsc_base) * tscFCvtt2n) / 2**32) + rnt_ns_base;

  *
  *

- *     ns = (((rdtsc - rnt_tsc_base) * 10e9) / tscFreq) + rnt_ns_base;
+ * We don't do so in order to use the same algorithm in 32- and 64-bit mode.
+ * When U32 goes away, we should reconsider.

  *
  * Since this routine is not synchronized and can be called in any context, 
  * we use a generation count to guard against seeing partially updated data.
  *
  * Since this routine is not synchronized and can be called in any context, 
  * we use a generation count to guard against seeing partially updated data.


@@ -135,41 +146,121 @@ ENTRY(_rtc_nanotime_adjust)
  * the generation is zero.
  *
  * unint64_t _rtc_nanotime_read(
  * the generation is zero.
  *
  * unint64_t _rtc_nanotime_read(

- *                     rtc_nanotime_t *rntp,           // %rdi
- *                     int            slow);           // %rsi
+ *                     rtc_nanotime_t *rntp);          // %rdi

  *
  */
 ENTRY(_rtc_nanotime_read)
  *
  */
 ENTRY(_rtc_nanotime_read)

-       test            %rsi,%rsi
-       jnz             Lslow
-               
-       /*
-        * Processor whose TSC frequency is faster than SLOW_TSC_THRESHOLD
-        */
+

        PAL_RTC_NANOTIME_READ_FAST()
 
        ret
        PAL_RTC_NANOTIME_READ_FAST()
 
        ret

+    
+/*
+ * extern uint64_t _rtc_tsc_to_nanoseconds(
+ *          uint64_t    value,              // %rdi
+ *          pal_rtc_nanotime_t *rntp);     // %rsi
+ *
+ * Converts TSC units to nanoseconds, using an abbreviated form of the above
+ * algorithm.  Note that while we could have simply used tmrCvt(value,tscFCvtt2n),
+ * which would avoid the need for this asm, doing so is a bit more risky since
+ * we'd be using a different algorithm with possibly different rounding etc.
+ */

 
 

-       /*
-        * Processor whose TSC frequency is not faster than SLOW_TSC_THRESHOLD
-        * But K64 doesn't support this...
-        */
-Lslow:
-       lea     1f(%rip),%rdi
-       xorb    %al,%al
-       call    EXT(panic)
-       hlt
-       .data
-1:     String  "_rtc_nanotime_read() - slow algorithm not supported"
+ENTRY(_rtc_tsc_to_nanoseconds)
+       movq    %rdi,%rax                       /* copy value (in TSC units) to convert */
+       movl    RNT_SHIFT(%rsi),%ecx
+       movl    RNT_SCALE(%rsi),%edx
+       shlq    %cl,%rax                        /* tscUnits << shift */
+       mulq    %rdx                            /* (tscUnits << shift) * scale */
+       shrdq   $32,%rdx,%rax                   /* %rdx:%rax >>= 32 */
+       ret
+    

 
 

+/*
+ *  typedef void (*thread_continue_t)(void *param, wait_result_t)
+ *
+ *     void call_continuation( thread_continue_t continuation,
+ *                                     void *param,
+ *                                         wait_result_t wresult,
+ *                          bool enable interrupts)
+ */

 
 Entry(call_continuation)
 
 Entry(call_continuation)

-       movq    %rdi,%rcx                       /* get continuation */
-       movq    %rsi,%rdi                       /* continuation param */
-       movq    %rdx,%rsi                       /* wait result */
+
+       movq    %rdi, %r12  /* continuation */
+    movq    %rsi, %r13  /* continuation param */
+    movq    %rdx, %r14  /* wait result */
+

        movq    %gs:CPU_KERNEL_STACK,%rsp       /* set the stack */
        xorq    %rbp,%rbp                       /* zero frame pointer */
        movq    %gs:CPU_KERNEL_STACK,%rsp       /* set the stack */
        xorq    %rbp,%rbp                       /* zero frame pointer */

+
+    test    %ecx, %ecx
+    jz 1f
+    mov     $1, %edi
+    call   _ml_set_interrupts_enabled
+1:
+
+       movq    %r12,%rcx                       /* continuation */
+       movq    %r13,%rdi                       /* continuation param */
+       movq    %r14,%rsi                       /* wait result */
+

        call    *%rcx                           /* call continuation */
        movq    %gs:CPU_ACTIVE_THREAD,%rdi
        call    EXT(thread_terminate)
 
        call    *%rcx                           /* call continuation */
        movq    %gs:CPU_ACTIVE_THREAD,%rdi
        call    EXT(thread_terminate)
 

+
+Entry(x86_init_wrapper)
+       xor     %rbp, %rbp
+       movq    %rsi, %rsp
+       callq   *%rdi
+
+#if CONFIG_VMX
+
+/*
+ *     __vmxon -- Enter VMX Operation
+ *     int __vmxon(addr64_t v);
+ */
+Entry(__vmxon)
+       FRAME
+       push    %rdi
+       
+       mov     $(VMX_FAIL_INVALID), %ecx
+       mov     $(VMX_FAIL_VALID), %edx
+       mov     $(VMX_SUCCEED), %eax
+       vmxon   (%rsp)
+       cmovcl  %ecx, %eax      /* CF = 1, ZF = 0 */
+       cmovzl  %edx, %eax      /* CF = 0, ZF = 1 */
+
+       pop     %rdi
+       EMARF
+       ret
+
+/*
+ *     __vmxoff -- Leave VMX Operation
+ *     int __vmxoff(void);
+ */
+Entry(__vmxoff)
+       FRAME
+       
+       mov     $(VMX_FAIL_INVALID), %ecx
+       mov     $(VMX_FAIL_VALID), %edx
+       mov     $(VMX_SUCCEED), %eax
+       vmxoff
+       cmovcl  %ecx, %eax      /* CF = 1, ZF = 0 */
+       cmovzl  %edx, %eax      /* CF = 0, ZF = 1 */
+
+       EMARF
+       ret
+
+#endif /* CONFIG_VMX */
+
+/*
+ *     mfence -- Memory Barrier
+ *     Use out-of-line assembly to get
+ *     standard x86-64 ABI guarantees
+ *     about what the caller's codegen
+ *     has in registers vs. memory
+ */
+Entry(do_mfence)
+       mfence
+       ret