xnu-1699.24.23.tar.gz

[apple/xnu.git] / bsd / crypto / aes / i386 / ExpandKeyForEncryption.s

/*      This file defines _aes_encrypt_key, _aes_encrypt_key128,
        _aes_encrypt_key192, and _aes_encrypt_key256.  It is designed to be
        included in another assembly file with the preprocessor #include directive,
        to benefit from some assembly-time calculations.

        Written by Eric Postpischil, January 2008.

        The comments here do not say much about the algorithm; the code just
        follows the FIPS-197 specification.  I recommend reading the specification
        before working with this code or examining the C code in the parent
        directory that illustrates key expansion.
*/


/*      Routines:

                _aes_encrypt_key.

                _aes_encrypt_key128, _aes_encrypt_key192, and _aes_encrypt_key256.

        Function:

                Expand the user's cipher key into the key schedule, as defined in
                Federal Information Processing Standards Publication 197 (FIPS-197),
                November 26, 2001.

        Input:

                Constant data:

                        The following names must be locally defined so the assembler
                        can calculate certain offsets.

                        static const Word _AESSubBytesWordTable[4][256].

                                _AESSubBytesWordTable[i][j] = SubBytes(j) << 8*i, where
                                SubBytes is defined in FIPS-197.  _AESSubBytesWordTable
                                differs from _AESEncryptTable in that it does not include
                                the MixColumn operation.  It is used in performing the last
                                round, which differs fromm the previous rounds in that it
                                does not include the MixColumn operation.

                        static const Byte _AESRcon[].

                                Round constants, beginning with AESRcon[1] for the first round
                                (AESRcon[0] is padding.)
        
                Arguments:

                        const uint8_t *Key

                                Address of user's cipher key.

                        int Length

                                Number of bytes (16, 24, or 32) or bits (128, 192, or 256) in
                                user's cipher key.

                                This argument is used with _aes_encrypt_key.  It is not
                                present for the other routines.  In those routines, Context
                                is the second argument.

                        aes_encrypt_ctx *Context

                                Structure to contain the expanded key beginning at offset
                                ContextKey and a four-byte "key length" beginning at offset
                                ContextKeyLength.  The "key length" is the number of bytes from
                                the start of the first round key to the start of the last round
                                key.  That is 16 less than the number of bytes in the entire
                                key.

        Output:

                The expanded key and the "key length" are written to *Context.

        Return:

                aes_rval        // -1 if "key length" is invalid.  0 otherwise.
*/

/* add AES HW detection and program branch if AES HW is detected cclee 3-12-10 */
#ifdef KERNEL
#include <i386/cpu_capabilities.h>
#else
#include <System/i386/cpu_capabilities.h>
#endif

        .text
        .globl _aes_encrypt_key
//      .private_extern _aes_encrypt_key
_aes_encrypt_key:

    // detect AES HW, cclee-3-13-10
#if defined __x86_64__
    movq    __cpu_capabilities@GOTPCREL(%rip), %rax                             // %rax -> __cpu_capabilities
    mov     (%rax), %eax                                                                                // %eax  = __cpu_capabilities
#else
#if defined KERNEL
    leal    __cpu_capabilities, %eax                                                    // %eax -> __cpu_capabilities
    mov     (%eax), %eax                                                                                // %eax  = __cpu_capabilities
#else
        mov    _COMM_PAGE_CPU_CAPABILITIES, %eax
#endif
#endif
    test    $(kHasAES), %eax                                                                    // __cpu_capabilities & kHasAES
    jne     _aes_encrypt_key_hw                                                                 // if AES HW detected, branch to _aes_encrypt_key_hw

#define dr              r0d                             // Dissection register.
#define drl             r0l                             // Low 8 bits of dissection register.
#define drh             r0h                             // Second-lowest 8 bits of dissection register.

#define t0              r1
#define t0d             r1d                             // Low 32 bits of t0.

#define offset  Arch(r5, r11)   // Address offset and loop sentinel.

#define R               r7                              // Address of round constant.
#define K               r7                              // User key pointer.
        // R and K overlap.

#define E               r6                              // Expanded key pointer.

#define ve0             %xmm0
#define ve1             %xmm1
#define ve2             %xmm2
#define ve3             %xmm3
#define vt3             %xmm4
#define vt2             %xmm5
#define vt1             %xmm6
#define vt0             %xmm7

#if defined __i386__
        #define LookupS(table, index)   \
                _AESSubBytesWordTable+(table)*TableSize(, index, 4)
#elif defined __x86_64__
        #define LookupS(table, index)   (table)*TableSize(STable, index, 4)
#endif

        /*      Save registers and set SaveSize to the number of bytes pushed onto the
                stack so far, including the caller's return address.
        */
        push    r3
        #if defined __i386__
                push    r5
                push    r6
                push    r7
                #define SaveSize        (5*4)
        #else
                #define SaveSize        (2*8)
        #endif

        /*      Number of bytes used for local variables:

                        8 16-byte spaces to save XMM registers.
        */
        #define LocalsSize      (8*16)

        #if 0 < LocalsSize
                // Padding to position stack pointer at a multiple of 16 bytes.
                #define Padding (15 & -(SaveSize + LocalsSize))
                sub             $Padding + LocalsSize, r4       // Allocate space on stack.
        #else
                #define Padding 0
        #endif

        /*      StackFrame is the number of bytes in our stack frame, from caller's
                stack pointer to ours (so it includes the return address).
        */
        #define StackFrame      (SaveSize + Padding + LocalsSize)

        // Save xmm registers.
        movaps  %xmm0, 0*16(r4)
        movaps  %xmm1, 1*16(r4)
        movaps  %xmm2, 2*16(r4)
        movaps  %xmm3, 3*16(r4)
        movaps  %xmm4, 4*16(r4)
        movaps  %xmm5, 5*16(r4)
        movaps  %xmm6, 6*16(r4)
        movaps  %xmm7, 7*16(r4)

#if defined __i386__

        // Define location of argument i.
        #define Argument(i)     StackFrame+4*(i)(r4)

        #define Nk              t0d

        // Load arguments.
        mov             Argument(2), E
        mov             Argument(1), Nk
        mov             Argument(0), K

#elif defined __x86_64__

        #define Nk              r9d                     // Number of words in key.
        mov             r6d, Nk                         // Move Nk argument out of way.
        mov             r2, E                           // Move E argument to common register.

#endif

        // Dispatch on key length.
        cmp             $128, Nk
        jge             2f
        shl             $3, Nk                          // Convert from bytes to bits.
        cmp             $128, Nk
2:
        je              EKeyHas4Words
        cmp             $192, Nk
        je              EKeyHas6Words
        cmp             $256, Nk
        je              EKeyHas8Words
        mov             $-1, r0                         // Return error.
        jmp             9f

// Stop using Nk.
#undef  Nk

        .globl _aes_encrypt_key128
//      .private_extern _aes_encrypt_key128
_aes_encrypt_key128:

        /*      Save registers and set SaveSize to the number of bytes pushed onto the
                stack so far, including the caller's return address.
        */
        push    r3
        #if defined __i386__
                push    r5
                push    r6
                push    r7
                #define SaveSize        (5*4)
        #else
                #define SaveSize        (2*8)
        #endif

        /*      Number of bytes used for local variables:

                        8 16-byte spaces to save XMM registers.
        */
        #define LocalsSize      (8*16)

        #if 0 < LocalsSize
                // Padding to position stack pointer at a multiple of 16 bytes.
                #define Padding (15 & -(SaveSize + LocalsSize))
                sub             $Padding + LocalsSize, r4       // Allocate space on stack.
        #else
                #define Padding 0
        #endif

        /*      StackFrame is the number of bytes in our stack frame, from caller's
                stack pointer to ours (so it includes the return address).
        */
        #define StackFrame      (SaveSize + Padding + LocalsSize)

        // Save xmm registers.
        movaps  %xmm0, 0*16(r4)
        movaps  %xmm1, 1*16(r4)
        movaps  %xmm2, 2*16(r4)
        movaps  %xmm3, 3*16(r4)
        movaps  %xmm4, 4*16(r4)
        movaps  %xmm5, 5*16(r4)
        movaps  %xmm6, 6*16(r4)
        movaps  %xmm7, 7*16(r4)

        #if defined __i386__

                // Load arguments.
                #define Argument(i)     StackFrame+4*(i)(r4)
                mov             Argument(1), E
                mov             Argument(0), K

        #endif

// Merge point for _aes_encrypt_key and _aes_encrypt_key128.
EKeyHas4Words:

#define e0      r2d
#define e1      r3d
#define e2      Arch(r5d, r11d)
#define e3      r7d

        // First words of expanded key are copied from user key.
        mov             0*4(K), e0
        mov             1*4(K), e1
        mov             2*4(K), e2
        mov             3*4(K), e3

        movl    $10*16, ContextKeyLength(E)     // Set "key length."

        #if 0 != ContextKey
                add             $ContextKey, E
        #endif

        // K cannot be used after we write to R, since they use the same register.

        // Cache round constants in output buffer.  The last is a sentinel.
        movb    $0x01,  1*16(E)
        movb    $0x02,  2*16(E)
        movb    $0x04,  3*16(E)
        movb    $0x08,  4*16(E)
        movb    $0x10,  5*16(E)
        movb    $0x20,  6*16(E)
        movb    $0x40,  7*16(E)
        movb    $0x80,  8*16(E)
        movb    $0x1b,  9*16(E)
        movb    $0x36, 10*16(E)

        #if defined __x86_64__

                #define STable  r8
                lea             _AESSubBytesWordTable(%rip), STable

        #endif

        // Store initial words of expanded key, which are copies of user's key.
        mov             e0, 0*4(E)
        mov             e1, 1*4(E)
        mov             e2, 2*4(E)
        mov             e3, 3*4(E)

1:
        mov             e3, dr                          // Put previous word into dissection register.

        // Perform SubWord(RotWord(dr)).
        movzx   drl, t0
        xor             LookupS(3, t0), e0              // Look up byte 0 in table 3.
        movzx   drh, t0d
        xor             LookupS(0, t0), e0              // Look up byte 1 in table 0.
        shr             $16, dr
        movzx   drl, t0d
        xor             LookupS(1, t0), e0              // Look up byte 2 in table 1.
        movzx   drh, t0d
        xor             LookupS(2, t0), e0              // Look up byte 3 in table 2.

        add             $4*4, E

        movzx   (E), t0d                                // Get cached round constant.
        xor             t0d, e0                                 // XOR with word from four words back.

        // Chain to successive words.
        mov             e0, 0*4(E)
        xor             e0, e1
        mov             e1, 1*4(E)
        xor             e1, e2
        mov             e2, 2*4(E)
        xor             e2, e3
        mov             e3, 3*4(E)

        cmp             $0x36, t0d                              // Was this the last round constant?

        jne             1b

        xor             r0, r0          // Return success.

9:
        // Pop stack and restore registers.
        movaps  7*16(r4), %xmm7
        movaps  6*16(r4), %xmm6
        movaps  5*16(r4), %xmm5
        movaps  4*16(r4), %xmm4
        movaps  3*16(r4), %xmm3
        movaps  2*16(r4), %xmm2
        movaps  1*16(r4), %xmm1
        movaps  0*16(r4), %xmm0
        #if 0 < LocalsSize
                add             $Padding + LocalsSize, r4
        #endif
        #if defined __i386__
                pop             r7
                pop             r6
                pop             r5
        #endif
        pop             r3

        ret


// Reset definitions for next case.
#undef  e0
#undef  e1
#undef  e2
#undef  e3

#undef  vt3
#undef  vt2
#define ve4     %xmm4
#define ve5     %xmm5


        .globl _aes_encrypt_key192
//      .private_extern _aes_encrypt_key192
_aes_encrypt_key192:

        /*      Save registers and set SaveSize to the number of bytes pushed onto the
                stack so far, including the caller's return address.
        */
        push    r3
        #if defined __i386__
                push    r5
                push    r6
                push    r7
                #define SaveSize        (5*4)
        #else
                #define SaveSize        (2*8)
        #endif

        /*      Number of bytes used for local variables:

                        8 16-byte spaces to save XMM registers.
        */
        #define LocalsSize      (8*16)

        #if 0 < LocalsSize
                // Padding to position stack pointer at a multiple of 16 bytes.
                #define Padding (15 & -(SaveSize + LocalsSize))
                sub             $Padding + LocalsSize, r4       // Allocate space on stack.
        #else
                #define Padding 0
        #endif

        /*      StackFrame is the number of bytes in our stack frame, from caller's
                stack pointer to ours (so it includes the return address).
        */
        #define StackFrame      (SaveSize + Padding + LocalsSize)

        // Save xmm registers.
        movaps  %xmm0, 0*16(r4)
        movaps  %xmm1, 1*16(r4)
        movaps  %xmm2, 2*16(r4)
        movaps  %xmm3, 3*16(r4)
        movaps  %xmm4, 4*16(r4)
        movaps  %xmm5, 5*16(r4)
        movaps  %xmm6, 6*16(r4)
        movaps  %xmm7, 7*16(r4)

        #if defined __i386__

                // Load arguments.
                #define Argument(i)     StackFrame+4*(i)(r4)
                mov             Argument(1), E
                mov             Argument(0), K

        #endif

// Merge point for _aes_encrypt_key and _aes_encrypt_key192.
EKeyHas6Words:

        // First words of expanded key are copied from user key.
        movd    0*4(K), ve0
        movd    1*4(K), ve1
        movd    2*4(K), ve2
        movd    3*4(K), ve3

        movl    $12*16, ContextKeyLength(E)     // Set "key length."

        #if 0 != ContextKey
                add             $ContextKey, E
        #endif

        movd    4*4(K), ve4
        movd    5*4(K), ve5

        // K cannot be used after we write to R, since they use the same register.

        #if defined __i386__

                lea             _AESRcon, R

        #elif defined __x86_64__

                lea             _AESRcon(%rip), R
                lea             _AESSubBytesWordTable(%rip), STable

        #endif

        /*      With a six-word key, there are twelve rounds (thirteen 16-byte key
                blocks).
        */
        mov             $-12*4*4, offset
        sub             offset, E

        // Store initial words of expanded key, which are copies of user's key.
        movd    ve0, 0*4(E, offset)
        movd    ve1, 1*4(E, offset)
        movd    ve2, 2*4(E, offset)
        movd    ve3, 3*4(E, offset)
        movd    ve4, 4*4(E, offset)
        movd    ve5, 5*4(E, offset)

/*      Jump into loop body.  The key expansion processes six four-byte words per
        iteration.  52 are needed in the key.  So only four are needed in the last
        iteration.
*/
        jmp             2f              
1:
        // Continue chaining to successive words.
        pxor    ve3, ve4
        movd    ve4, 4*4(E, offset)
        pxor    ve4, ve5
        movd    ve5, 5*4(E, offset)
2:
        add             $1, R                           // Advance pointer.
        movd    ve5, dr                         // Put previous word into dissection register.
        movzx   (R), t0                         // Get round constant.
        movd    t0d, vt1
        pxor    vt1, ve0                        // XOR with word from six words back.

        // Perform SubWord(RotWord(dr)).
        movzx   drl, t0d
        movd    LookupS(3, t0), vt0             // Look up byte 0 in table 3.
        movzx   drh, t0d
        movd    LookupS(0, t0), vt1             // Look up byte 1 in table 0.
        shr             $16, dr
        movzx   drl, t0d
        pxor    vt1, vt0
        pxor    vt0, ve0
        movd    LookupS(1, t0), vt0             // Look up byte 2 in table 1.
        movzx   drh, t0d
        movd    LookupS(2, t0), vt1             // Look up byte 3 in table 2.
        pxor    vt1, vt0
        pxor    vt0, ve0

        add             $6*4, offset

        // Chain to successive words.
        movd    ve0, 0*4(E, offset)
        pxor    ve0, ve1
        movd    ve1, 1*4(E, offset)
        pxor    ve1, ve2
        movd    ve2, 2*4(E, offset)
        pxor    ve2, ve3
        movd    ve3, 3*4(E, offset)

        jne             1b

        xor             r0, r0          // Return success.

        // Pop stack and restore registers.
        movaps  7*16(r4), %xmm7
        movaps  6*16(r4), %xmm6
        movaps  5*16(r4), %xmm5
        movaps  4*16(r4), %xmm4
        movaps  3*16(r4), %xmm3
        movaps  2*16(r4), %xmm2
        movaps  1*16(r4), %xmm1
        movaps  0*16(r4), %xmm0
        #if 0 < LocalsSize
                add             $Padding + LocalsSize, r4
        #endif
        #if defined __i386__
                pop             r7
                pop             r6
                pop             r5
        #endif
        pop             r3

        ret


// Reset definitions for next case.
#undef  ve4
#undef  ve5
#define vt3     %xmm4
#define vt2     %xmm5


        .globl _aes_encrypt_key256
//      .private_extern _aes_encrypt_key256
_aes_encrypt_key256:

        /*      Save registers and set SaveSize to the number of bytes pushed onto the
                stack so far, including the caller's return address.
        */
        push    r3
        #if defined __i386__
                push    r5
                push    r6
                push    r7
                #define SaveSize        (5*4)
        #else
                #define SaveSize        (2*8)
        #endif

        /*      Number of bytes used for local variables:

                        8 16-byte spaces to save XMM registers.
        */
        #define LocalsSize      (8*16)

        #if 0 < LocalsSize
                // Padding to position stack pointer at a multiple of 16 bytes.
                #define Padding (15 & -(SaveSize + LocalsSize))
                sub             $Padding + LocalsSize, r4       // Allocate space on stack.
        #else
                #define Padding 0
        #endif

        /*      StackFrame is the number of bytes in our stack frame, from caller's
                stack pointer to ours (so it includes the return address).
        */
        #define StackFrame      (SaveSize + Padding + LocalsSize)

        // Save xmm registers.
        movaps  %xmm0, 0*16(r4)
        movaps  %xmm1, 1*16(r4)
        movaps  %xmm2, 2*16(r4)
        movaps  %xmm3, 3*16(r4)
        movaps  %xmm4, 4*16(r4)
        movaps  %xmm5, 5*16(r4)
        movaps  %xmm6, 6*16(r4)
        movaps  %xmm7, 7*16(r4)

        #if defined __i386__

                // Load arguments.
                #define Argument(i)     StackFrame+4*(i)(r4)
                mov             Argument(1), E
                mov             Argument(0), K

        #endif

// Merge point for _aes_encrypt_key and _aes_encrypt_key256.
EKeyHas8Words:

        // First words of expanded key are copied from user key.
        movd    0*4(K), ve0
        movd    1*4(K), ve1
        movd    2*4(K), ve2
        movd    3*4(K), ve3

        movl    $14*16, ContextKeyLength(E)     // Set "key length."

        #if 0 != ContextKey
                add             $ContextKey, E
        #endif

        // Store initial words of expanded key, which are copies of user's key.
        movd    ve0, 0*4(E)
        movd    ve1, 1*4(E)
        movd    ve2, 2*4(E)
        movd    ve3, 3*4(E)
        movd    4*4(K), ve0
        movd    5*4(K), ve1
        movd    6*4(K), ve2
        movd    7*4(K), ve3

        // K cannot be used after we write to R, since they use the same register.

        #if defined __i386__

                lea             _AESRcon, R

        #elif defined __x86_64__

                lea             _AESRcon(%rip), R
                lea             _AESSubBytesWordTable(%rip), STable

        #endif

        /*      With an eight-word key, there are fourteen rounds (fifteen 16-byte key
                blocks).
        */
        mov             $-14*4*4, offset
        sub             offset, E

        // Store initial words of expanded key, which are copies of user's key.
        movd    ve0, 4*4(E, offset)
        movd    ve1, 5*4(E, offset)
        movd    ve2, 6*4(E, offset)
        movd    ve3, 7*4(E, offset)

/*      Jump into loop body.  The key expansion processes eight four-byte words per
        iteration.  60 are needed in the key.  So only four are needed in the last
        iteration.
*/
        jmp             2f              
1:
        movd    ve3, dr                         // Put previous word into dissection register.

        /*      Get word from eight words back (it is four words back from where E
                currently points, and we use it to prepare the value to be stored
                four words beyond where E currently points).
        */
        movd    -4*4(E, offset), ve0

        // Perform SubWord(dr).
        movzx   drl, t0
        movd    LookupS(0, t0), vt0             // Look up byte 0 in table 0.
        movzx   drh, t0d
        movd    LookupS(1, t0), vt1             // Look up byte 1 in table 1.
        shr             $16, dr
        movzx   drl, t0d
        movd    LookupS(2, t0), vt2             // Look up byte 2 in table 2.
        movzx   drh, t0d
        movd    LookupS(3, t0), vt3             // Look up byte 3 in table 3.
        pxor    vt1, vt0
        pxor    vt3, vt2
        pxor    vt0, ve0
        pxor    vt2, ve0

        movd    -3*4(E, offset), ve1    // Get words from eight words back.
        movd    -2*4(E, offset), ve2
        movd    -1*4(E, offset), ve3

        // Chain to successive words.
        movd    ve0, 4*4(E, offset)
        pxor    ve0, ve1
        movd    ve1, 5*4(E, offset)
        pxor    ve1, ve2
        movd    ve2, 6*4(E, offset)
        pxor    ve2, ve3
        movd    ve3, 7*4(E, offset)

2:
        add             $1, R                           // Advance pointer.
        movd    ve3, dr                         // Put previous word into dissection register.
        movzx   (R), t0d                        // Get round constant.
        movd    t0d, vt1
        movd    0*4(E, offset), ve0     // Get word from eight words back.
        pxor    vt1, ve0

        // Perform SubWord(RotWord(dr)).
        movzx   drl, t0
        movd    LookupS(3, t0), vt0             // Look up byte 0 in table 3.
        movzx   drh, t0d
        movd    LookupS(0, t0), vt1             // Look up byte 1 in table 0.
        shr             $16, dr
        movzx   drl, t0d
        movd    LookupS(1, t0), vt2             // Look up byte 2 in table 1.
        movzx   drh, t0d
        movd    LookupS(2, t0), vt3             // Look up byte 3 in table 2.
        pxor    vt1, vt0
        pxor    vt3, vt2
        pxor    vt0, ve0
        pxor    vt2, ve0

        movd    1*4(E, offset), ve1
        movd    2*4(E, offset), ve2
        movd    3*4(E, offset), ve3

        add             $8*4, offset

        // Chain to successive words.
        movd    ve0, 0*4(E, offset)
        pxor    ve0, ve1
        movd    ve1, 1*4(E, offset)
        pxor    ve1, ve2
        movd    ve2, 2*4(E, offset)
        pxor    ve2, ve3
        movd    ve3, 3*4(E, offset)

        jne             1b

        xor             r0, r0          // Return success.

        // Pop stack and restore registers.
        movaps  7*16(r4), %xmm7
        movaps  6*16(r4), %xmm6
        movaps  5*16(r4), %xmm5
        movaps  4*16(r4), %xmm4
        movaps  3*16(r4), %xmm3
        movaps  2*16(r4), %xmm2
        movaps  1*16(r4), %xmm1
        movaps  0*16(r4), %xmm0
        #if 0 < LocalsSize
                add             $Padding + LocalsSize, r4
        #endif
        #if defined __i386__
                pop             r7
                pop             r6
                pop             r5
        #endif
        pop             r3

        ret


#undef  Address
#undef  Argument
#undef  E
#undef  K
#undef  LocalsSize
#undef  LookupS
#undef  Padding
#undef  R
#undef  SaveSize
#undef  STable
#undef  StackFrame
#undef  dr
#undef  drh
#undef  drl
#undef  offset
#undef  t0
#undef  t0d
#undef  ve0
#undef  ve1
#undef  ve2
#undef  ve3
#undef  vt0
#undef  vt1
#undef  vt2
#undef  vt3