xnu-1228.12.14.tar.gz

[apple/xnu.git] / osfmk / i386 / commpage / commpage_mach_absolute_time.s

diff --git a/osfmk/i386/commpage/commpage_mach_absolute_time.s b/osfmk/i386/commpage/commpage_mach_absolute_time.s
index 3427958ed0ad9fad0790586e5f20958db39e2b78..f10baef8b9128d49e3b3a751c11c6f3611137a7e 100644 (file)
--- a/osfmk/i386/commpage/commpage_mach_absolute_time.s
+++ b/osfmk/i386/commpage/commpage_mach_absolute_time.s
@@ -1,23 +1,29 @@
 /*
 /*

- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 
 #include <sys/appleapiopts.h>
  */
 
 #include <sys/appleapiopts.h>


@@ -31,89 +37,147 @@
         .align  2, 0x90
 
 Lmach_absolute_time:
         .align  2, 0x90
 
 Lmach_absolute_time:

-

        int     $0x3
        ret
 
        int     $0x3
        ret
 

-       COMMPAGE_DESCRIPTOR(mach_absolute_time,_COMM_PAGE_ABSOLUTE_TIME,1,0)
+       COMMPAGE_DESCRIPTOR(mach_absolute_time,_COMM_PAGE_ABSOLUTE_TIME,0,0)

 
 

+ 
+/* return nanotime in %edx:%eax */

 
 Lnanotime:
 
 Lnanotime:

-
-       pushl   %ebx
-       pushl   %esi
-       pushl   %edi

        pushl   %ebp
        pushl   %ebp

-       movl    $(_COMM_PAGE_NANOTIME_INFO), %esi
+       movl    %esp,%ebp
+       pushl   %esi
+       pushl   %ebx
+
+0:
+       movl    _COMM_PAGE_NT_GENERATION,%esi   /* get generation (0 if being changed) */
+       testl   %esi,%esi                       /* if being updated, loop until stable */
+       jz      0b

 
 

+       lfence
+       rdtsc                                   /* get TSC in %edx:%eax */
+       lfence
+
+       subl    _COMM_PAGE_NT_TSC_BASE,%eax
+       sbbl    _COMM_PAGE_NT_TSC_BASE+4,%edx
+
+       movl    _COMM_PAGE_NT_SCALE,%ecx
+
+       movl    %edx,%ebx
+       mull    %ecx
+       movl    %ebx,%eax
+       movl    %edx,%ebx
+       mull    %ecx
+       addl    %ebx,%eax
+       adcl    $0,%edx
+
+       addl    _COMM_PAGE_NT_NS_BASE,%eax
+       adcl    _COMM_PAGE_NT_NS_BASE+4,%edx
+
+       cmpl    _COMM_PAGE_NT_GENERATION,%esi   /* have the parameters changed? */
+       jne     0b                              /* yes, loop until stable */
+
+       popl    %ebx
+       popl    %esi
+       popl    %ebp
+       ret
+
+       COMMPAGE_DESCRIPTOR(nanotime,_COMM_PAGE_NANOTIME,0,kSlow)
+
+
+/* nanotime routine for machines slower than ~1Gz (SLOW_TSC_THRESHOLD) */
+Lnanotime_slow:
+       push    %ebp
+       mov     %esp,%ebp
+       push    %esi
+       push    %edi
+       push    %ebx
+
+0:
+       movl    _COMM_PAGE_NT_GENERATION,%esi
+       testl   %esi,%esi                       /* if generation is 0, data being changed */
+       jz      0b                              /* so loop until stable */
+
+       lfence
+       rdtsc                                   /* get TSC in %edx:%eax */
+       lfence
+       subl    _COMM_PAGE_NT_TSC_BASE,%eax
+       sbbl    _COMM_PAGE_NT_TSC_BASE+4,%edx
+
+       pushl   %esi                            /* save generation */

        /*
        /*

-        * The nanotime info consists of:
-        *      - base_tsc      64-bit timestamp register value
-        *      - base_ns       64-bit corresponding nanosecond uptime value
-        *      - scale         32-bit current scale multiplier
-        *      - shift         32-bit current shift divider
-        *      - check_tsc     64-bit timestamp check value
-        *
-        * This enables an timestamp register's value, tsc, to be converted
-        * into a nanosecond nanotime value, ns:
-        *
-        *      ns = base_ns + ((tsc - base_tsc) * scale >> shift)
-        *
-        * The kernel updates this every tick or whenever a performance
-        * speed-step changes the scaling. To avoid locking, a duplicated
-        * sequence counting scheme is used. The base_tsc value is updated
-        * whenever the info starts to be changed, and check_tsc is updated
-        * to the same value at the end of the update. The regularity of
-        * update ensures that (tsc - base_tsc) is a 32-bit quantity.
-        * When a conversion is performed, we read base_tsc before we start
-        * and check_tsc at the end -- if there's a mis-match we repeat.
-        * It's sufficient to compare only the low-order 32-bits. 
+        * Do the math to convert tsc ticks to nanoseconds.  We first
+        * do long multiply of 1 billion times the tsc.  Then we do
+        * long division by the tsc frequency

         */
         */

-
+       mov     $1000000000, %ecx               /* number of nanoseconds in a second */
+       mov     %edx, %ebx
+       mul     %ecx
+       mov     %edx, %edi
+       mov     %eax, %esi
+       mov     %ebx, %eax
+       mul     %ecx
+       add     %edi, %eax
+       adc     $0, %edx                        /* result in edx:eax:esi */
+       mov     %eax, %edi
+       mov     _COMM_PAGE_NT_SHIFT,%ecx        /* overloaded as the low 32 tscFreq */
+       xor     %eax, %eax
+       xchg    %edx, %eax
+       div     %ecx
+       xor     %eax, %eax
+       mov     %edi, %eax
+       div     %ecx
+       mov     %eax, %ebx
+       mov     %esi, %eax
+       div     %ecx
+       mov     %ebx, %edx                      /* result in edx:eax */
+       popl    %esi                            /* recover generation */
+
+       add     _COMM_PAGE_NT_NS_BASE,%eax
+       adc     _COMM_PAGE_NT_NS_BASE+4,%edx
+
+       cmpl    _COMM_PAGE_NT_GENERATION,%esi   /* have the parameters changed? */
+       jne     0b                              /* yes, loop until stable */
+
+       pop     %ebx
+       pop     %edi
+       pop     %esi
+       pop     %ebp
+       ret                                     /* result in edx:eax */
+
+       COMMPAGE_DESCRIPTOR(nanotime_slow,_COMM_PAGE_NANOTIME,kSlow,0)
+
+
+/* The 64-bit version.  We return the 64-bit nanotime in %rax,
+ * and by convention we must preserve %r9, %r10, and %r11.
+ */
+       .text
+       .align  2
+       .code64
+Lnanotime_64:                                  // NB: must preserve r9, r10, and r11
+       pushq   %rbp                            // set up a frame for backtraces
+       movq    %rsp,%rbp
+       movq    $_COMM_PAGE_32_TO_64(_COMM_PAGE_TIME_DATA_START),%rsi

 1:
 1:

-       //
-       //  Read nanotime info and stash in registers.
-       //
-       movl    NANOTIME_BASE_TSC(%esi), %ebx   // ebx := lo(base_tsc)
-       movl    NANOTIME_BASE_NS(%esi), %ebp
-       movl    NANOTIME_BASE_NS+4(%esi), %edi  // edi:ebp := base_ns
-       movl    NANOTIME_SHIFT(%esi), %ecx      // ecx := shift
-       //
-       // Read timestamp register (tsc) and calculate delta.
-       //
+       movl    _NT_GENERATION(%rsi),%r8d       // get generation
+       testl   %r8d,%r8d                       // if 0, data is being changed...
+       jz      1b                              // ...so loop until stable
+       lfence

        rdtsc                                   // edx:eax := tsc
        rdtsc                                   // edx:eax := tsc

-       subl    %ebx, %eax                      // eax := (tsc - base_tsc)
-       movl    NANOTIME_SCALE(%esi), %edx      // edx := shift
-       //
-       // Check for consistency and re-read if necessary.
-       //
-       cmpl    NANOTIME_CHECK_TSC(%esi), %ebx
+       lfence
+       shlq    $32,%rdx                        // rax := ((edx << 32) | eax), ie 64-bit tsc
+       orq     %rdx,%rax
+       subq    _NT_TSC_BASE(%rsi), %rax        // rax := (tsc - base_tsc)
+       movl    _NT_SCALE(%rsi),%ecx
+       mulq    %rcx                            // rdx:rax := (tsc - base_tsc) * scale
+       shrdq   $32,%rdx,%rax                   // _COMM_PAGE_NT_SHIFT is always 32
+       addq    _NT_NS_BASE(%rsi),%rax          // (((tsc - base_tsc) * scale) >> 32) + ns_base
+       
+       cmpl    _NT_GENERATION(%rsi),%r8d       // did the data change during computation?

        jne     1b
        jne     1b

-
-       //
-       // edx:eax := ((tsc - base_tsc) * scale)
-       //
-       mull    %edx
-
-       //
-       // eax := ((tsc - base_tsc) * scale >> shift)
-       //
-       shrdl   %cl, %edx, %eax
-       andb    $32, %cl
-       cmovnel %edx, %eax              // %eax := %edx if shift == 32
-       xorl    %edx, %edx
-
-       //
-       // Add base_ns: 
-       // edx:eax = (base_ns + ((tsc - base_tsc) * scale >> shift))
-       //
-       addl    %ebp, %eax
-       adcl    %edi, %edx
-
-       popl    %ebp
-       popl    %edi
-       popl    %esi
-       popl    %ebx
+       popq    %rbp

        ret
 
        ret
 

-       COMMPAGE_DESCRIPTOR(nanotime,_COMM_PAGE_NANOTIME,1,0)
+       COMMPAGE_DESCRIPTOR(nanotime_64,_COMM_PAGE_NANOTIME,0,kSlow)