xnu-1699.26.8.tar.gz

[apple/xnu.git] / osfmk / i386 / machine_routines_asm.s

diff --git a/osfmk/i386/machine_routines_asm.s b/osfmk/i386/machine_routines_asm.s
index 02b2e6cd09dc7e089fbe8fd6875e7667109bbcd7..0e3d9fb68505a8e76a9e959d9e6237fb82a49d84 100644 (file)
--- a/osfmk/i386/machine_routines_asm.s
+++ b/osfmk/i386/machine_routines_asm.s
@@ -1,52 +1,39 @@
 /*
 /*

- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2010 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * This file contains Original Code and/or Modifications of Original Code 
- * as defined in and that are subject to the Apple Public Source License 
- * Version 2.0 (the 'License'). You may not use this file except in 
- * compliance with the License.  The rights granted to you under the 
- * License may not be used to create, or enable the creation or 
- * redistribution of, unlawful or unlicensed copies of an Apple operating 
- * system, or to circumvent, violate, or enable the circumvention or 
- * violation of, any terms of an Apple operating system software license 
- * agreement.
- *
- * Please obtain a copy of the License at 
- * http://www.opensource.apple.com/apsl/ and read it before using this 
- * file.
- *
- * The Original Code and all software distributed under the License are 
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 
- * Please see the License for the specific language governing rights and 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and

  * limitations under the License.
  * limitations under the License.

- *
- * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
  
 #include <i386/asm.h>
  */
  
 #include <i386/asm.h>

-#include <i386/proc_reg.h>
+#include <i386/apic.h>

 #include <i386/eflags.h>
 #include <i386/eflags.h>

-       
+#include <i386/rtclock_asm.h>

 #include <i386/postcode.h>
 #include <i386/postcode.h>

-#include <i386/apic.h>
+#include <i386/proc_reg.h>

 #include <assym.s>
 
 #include <assym.s>
 

-#define        PA(addr)        (addr)
-#define        VA(addr)        (addr)
-       
-/*
- * GAS won't handle an intersegment jump with a relocatable offset.
- */
-#define        LJMP(segment,address)   \
-       .byte   0xea            ;\
-       .long   address         ;\
-       .word   segment
-

 /*
 **      ml_get_timebase()
 **
 /*
 **      ml_get_timebase()
 **


@@ -59,14 +46,15 @@ ENTRY(ml_get_timebase)
 
                        movl    S_ARG0, %ecx
                        
 
                        movl    S_ARG0, %ecx
                        

+                       lfence

                        rdtsc
                        rdtsc

+                       lfence

                        
                        movl    %edx, 0(%ecx)
                        movl    %eax, 4(%ecx)
                        
                        ret
 
                        
                        movl    %edx, 0(%ecx)
                        movl    %eax, 4(%ecx)
                        
                        ret
 

-

 /*
  *     Convert between various timer units 
  *
 /*
  *     Convert between various timer units 
  *


@@ -144,61 +132,180 @@ LEXT(tmrCvt)
                        popl    %ebx                                    // Restore a volatile
                        popl    %ebp                                    // Restore a volatile
 
                        popl    %ebx                                    // Restore a volatile
                        popl    %ebp                                    // Restore a volatile
 

-                       ret                                                             // Leave...
+                       ret                                             // Leave...

 
 

-                       .globl  EXT(rtc_nanotime_store)
-                       .align  FALIGN

 
 

-LEXT(rtc_nanotime_store)
-               push    %ebp
-               mov             %esp,%ebp
+/* void  _rtc_nanotime_adjust( 
+               uint64_t         tsc_base_delta,
+               rtc_nanotime_t  *dst);
+*/
+       .globl  EXT(_rtc_nanotime_adjust)
+       .align  FALIGN

 
 

-               mov             32(%ebp),%edx
+LEXT(_rtc_nanotime_adjust)
+       mov     12(%esp),%edx                   /* ptr to rtc_nanotime_info */
+       
+       movl    RNT_GENERATION(%edx),%ecx       /* get current generation */
+       movl    $0,RNT_GENERATION(%edx)         /* flag data as being updated */

 
 

-               mov             8(%ebp),%eax
-               mov             %eax,RNT_TSC_BASE(%edx)
-               mov             12(%ebp),%eax
-               mov             %eax,RNT_TSC_BASE+4(%edx)
+       movl    4(%esp),%eax                    /* get lower 32-bits of delta */
+       addl    %eax,RNT_TSC_BASE(%edx)
+       adcl    $0,RNT_TSC_BASE+4(%edx)         /* propagate carry */

 
 

-               mov             24(%ebp),%eax
-               mov             %eax,RNT_SCALE(%edx)
+       incl    %ecx                            /* next generation */
+       jnz     1f
+       incl    %ecx                            /* skip 0, which is a flag */
+1:     movl    %ecx,RNT_GENERATION(%edx)       /* update generation and make usable */

 
 

-               mov             28(%ebp),%eax
-               mov             %eax,RNT_SHIFT(%edx)
+       ret

 
 

-               mov             16(%ebp),%eax
-               mov             %eax,RNT_NS_BASE(%edx)
-               mov             20(%ebp),%eax
-               mov             %eax,RNT_NS_BASE+4(%edx)

 
 

-               pop             %ebp
+/* unint64_t _rtc_nanotime_read( rtc_nanotime_t *rntp, int slow );
+ *
+ * This is the same as the commpage nanotime routine, except that it uses the
+ * kernel internal "rtc_nanotime_info" data instead of the commpage data.  The two copies
+ * of data (one in the kernel and one in user space) are kept in sync by rtc_clock_napped().
+ *
+ * Warning!  There is another copy of this code in osfmk/i386/locore.s.  The
+ * two versions must be kept in sync with each other!
+ *
+ * There are actually two versions of the algorithm, one each for "slow" and "fast"
+ * processors.  The more common "fast" algorithm is:
+ *
+ *     nanoseconds = (((rdtsc - rnt_tsc_base) * rnt_tsc_scale) / 2**32) - rnt_ns_base;
+ *
+ * Of course, the divide by 2**32 is a nop.  rnt_tsc_scale is a constant computed during initialization:
+ *
+ *     rnt_tsc_scale = (10e9 * 2**32) / tscFreq;
+ *
+ * The "slow" algorithm uses long division:
+ *
+ *     nanoseconds = (((rdtsc - rnt_tsc_base) * 10e9) / tscFreq) - rnt_ns_base;
+ *
+ * Since this routine is not synchronized and can be called in any context, 
+ * we use a generation count to guard against seeing partially updated data.  In addition,
+ * the _rtc_nanotime_store() routine -- just above -- zeroes the generation before
+ * updating the data, and stores the nonzero generation only after all other data has been
+ * stored.  Because IA32 guarantees that stores by one processor must be seen in order
+ * by another, we can avoid using a lock.  We spin while the generation is zero.
+ *
+ * In accordance with the ABI, we return the 64-bit nanotime in %edx:%eax.
+ */
+ 
+               .globl  EXT(_rtc_nanotime_read)
+               .align  FALIGN
+LEXT(_rtc_nanotime_read)
+               pushl           %ebp
+               movl            %esp,%ebp
+               pushl           %esi
+               pushl           %edi
+               pushl           %ebx
+               movl            8(%ebp),%edi                            /* get ptr to rtc_nanotime_info */
+               movl            12(%ebp),%eax                           /* get "slow" flag */
+               testl           %eax,%eax
+               jnz             Lslow
+               
+               /* Processor whose TSC frequency is faster than SLOW_TSC_THRESHOLD */
+               PAL_RTC_NANOTIME_READ_FAST()
+
+               popl            %ebx
+               popl            %edi
+               popl            %esi
+               popl            %ebp

                ret
 
                ret
 

-                       .globl  EXT(rtc_nanotime_load)
-                       .align  FALIGN
+               /* Processor whose TSC frequency is slower than or equal to SLOW_TSC_THRESHOLD */
+Lslow:
+               movl            RNT_GENERATION(%edi),%esi               /* get generation (0 if being changed) */
+               testl           %esi,%esi                               /* if being changed, loop until stable */
+               jz              Lslow
+               pushl           %esi                                    /* save generation */
+               pushl           RNT_SHIFT(%edi)                         /* save low 32 bits of tscFreq */
+
+               lfence
+               rdtsc                                                   /* get TSC in %edx:%eax */
+               lfence
+               subl            RNT_TSC_BASE(%edi),%eax
+               sbbl            RNT_TSC_BASE+4(%edi),%edx

 
 

-LEXT(rtc_nanotime_load)
-               push    %ebp
-               mov             %esp,%ebp
+               /*
+               * Do the math to convert tsc ticks to nanoseconds.  We first
+               * do long multiply of 1 billion times the tsc.  Then we do
+               * long division by the tsc frequency
+               */
+               mov             $1000000000, %ecx                       /* number of nanoseconds in a second */
+               mov             %edx, %ebx
+               mul             %ecx
+               mov             %edx, %edi
+               mov             %eax, %esi
+               mov             %ebx, %eax
+               mul             %ecx
+               add             %edi, %eax
+               adc             $0, %edx                                /* result in edx:eax:esi */
+               mov             %eax, %edi
+               popl            %ecx                                    /* get low 32 tscFreq */
+               xor             %eax, %eax
+               xchg            %edx, %eax
+               div             %ecx
+               xor             %eax, %eax
+               mov             %edi, %eax
+               div             %ecx
+               mov             %eax, %ebx
+               mov             %esi, %eax
+               div             %ecx
+               mov             %ebx, %edx                              /* result in edx:eax */
+               
+               movl            8(%ebp),%edi                            /* recover ptr to rtc_nanotime_info */
+               popl            %esi                                    /* recover generation */

 
 

-               mov             8(%ebp),%ecx
-               mov             12(%ebp),%edx
+               addl            RNT_NS_BASE(%edi),%eax
+               adcl            RNT_NS_BASE+4(%edi),%edx

 
 

-               mov             RNT_TSC_BASE(%ecx),%eax
-               mov             %eax,RNT_TSC_BASE(%edx)
-               mov             RNT_TSC_BASE+4(%ecx),%eax
-               mov             %eax,RNT_TSC_BASE+4(%edx)
+               cmpl            RNT_GENERATION(%edi),%esi               /* have the parameters changed? */
+               jne             Lslow                                   /* yes, loop until stable */

 
 

-               mov             RNT_SCALE(%ecx),%eax
-               mov             %eax,RNT_SCALE(%edx)
+               pop             %ebx
+               pop             %edi
+               pop             %esi
+               pop             %ebp
+               ret                                                     /* result in edx:eax */

 
 

-               mov             RNT_SHIFT(%ecx),%eax
-               mov             %eax,RNT_SHIFT(%edx)

 
 

-               mov             RNT_NS_BASE(%ecx),%eax
-               mov             %eax,RNT_NS_BASE(%edx)
-               mov             RNT_NS_BASE+4(%ecx),%eax
-               mov             %eax,RNT_NS_BASE+4(%edx)

 
 

-               pop             %ebp
-               ret
+/*
+ * Timing routines.
+ */
+Entry(timer_update)
+       movl    4(%esp),%ecx
+       movl    8(%esp),%eax
+       movl    12(%esp),%edx
+       movl    %eax,TIMER_HIGHCHK(%ecx)
+       movl    %edx,TIMER_LOW(%ecx)
+       movl    %eax,TIMER_HIGH(%ecx)
+       ret
+
+Entry(timer_grab)
+       movl    4(%esp),%ecx
+0:     movl    TIMER_HIGH(%ecx),%edx
+       movl    TIMER_LOW(%ecx),%eax
+       cmpl    TIMER_HIGHCHK(%ecx),%edx
+       jne     0b
+       ret
+
+
+Entry(call_continuation)
+       movl    S_ARG0,%eax                     /* get continuation */
+       movl    S_ARG1,%edx                     /* continuation param */
+       movl    S_ARG2,%ecx                     /* wait result */
+       movl    %gs:CPU_KERNEL_STACK,%esp       /* pop the stack */
+       xorl    %ebp,%ebp                       /* zero frame pointer */
+       subl    $8,%esp                         /* align the stack */
+       pushl   %ecx
+       pushl   %edx
+       call    *%eax                           /* call continuation */
+       addl    $16,%esp
+       movl    %gs:CPU_ACTIVE_THREAD,%eax
+       pushl   %eax
+       call    EXT(thread_terminate)
+
+