xnu-1699.24.23.tar.gz

[apple/xnu.git] / bsd / crypto / aes / i386 / ExpandKeyForDecryption.s

/*      This file defines _aes_decrypt_key, _aes_decrypt_key128,
        _aes_decrypt_key192, and _aes_decrypt_key256.  It is designed to be
        included in another assembly file with the preprocessor #include directive,
        to benefit from some assembly-time calculations.

        Written by Eric Postpischil, January 2008.

        The comments here do not say much about the algorithm; the code just
        follows the FIPS-197 specification.  I recommend reading the specification
        before working with this code or examining the C code in the parent
        directory that illustrates key expansion.

        One complication is that this routine both expands the key and applies
        InvMixColumn to most of the words in the expanded key.  This modifies the
        key for use with the Equivalent Inverse Cipher.

        During key expansion, there are sequences of four or six words that are
        produced like this:

                E[i+0] = E[i+0-Nk] ^ f(E[i-1]), where f is some function.
                E[i+1] = E[i+1-Nk] ^ E[i+0].
                E[i+2] = E[i+2-Nk] ^ E[i+1].
                E[i+3] = E[i+3-Nk] ^ E[i+2].

        When Nk is four or eight, the sequence stops there.  When it is six, it
        goes on for two more words.  Let I be the InvMixColumn function.  for the
        Equivalent Inverse Cipher, we want to store I(E[i+0]), I(E[i+1]),
        I(E[i+2]), I(E[i+3]) (and two more when Nk is six).  However, we do not
        need to calculate I four times.  In AES' finite field, I is a linear
        combination of the four bytes of its input.  The ^ operation on the bits
        that represent field elements is an addition in the Galois field.  So
        I(a ^ b) = I(a) ^ I(b).  Then we have:

                I(E[i+0]) = I(E[i+0-Nk] ^ f(E[i-1])) = I(E[i+0-Nk]) ^ I(f(E[i-1])).
                I(E[i+1]) = I(E[i+1-Nk]) ^ I(E[i+0]).
                I(E[i+2]) = I(E[i+2-Nk]) ^ I(E[i+1]).
                I(E[i+3]) = I(E[i+3-Nk]) ^ I(E[i+2]).

        To compute this, we compute I(f(E[i-1])) and XOR it with the previously
        stored E[i+0-Nk])) to get I(E[i+0])).  Then we XOR that with the previously
        stored E[i+1-Nk])) to get I(E[i+1])), and so on.

        Note that to compute I(f(E[i-1])), we need to have E[i-1].  So we have to
        compute the pre-InvMixColumn words of the expanded key; it is not
        sufficient to have the post-InvMixColumn words.
*/


/*      Routine:

                _aes_decrypt_key.

                _aes_decrypt_key128, _aes_decrypt_key192, and _aes_decrypt_key256.

        Function:

                Expand the user's cipher key into the key schedule, as defined in
                Federal Information Processing Standards Publication 197 (FIPS-197),
                November 26, 2001.

                For decryption, the key is modified as shown in Figure 15 in FIPS-197,
                to support the Equivalent Inverse Cipher.

        Input:

                Constant data:

                        The following names must be locally defined so the assembler
                        can calculate certain offsets.

                        static const Word _AESSubBytesWordTable[4][256].

                                _AESSubBytesWordTable[i][j] = SubBytes(j) << 8*i, where
                                SubBytes is defined in FIPS-197.  _AESSubBytesWordTable
                                differs from _AESEncryptTable in that it does not include
                                the MixColumn operation.  It is used in performing the last
                                round, which differs fromm the previous rounds in that it
                                does not include the MixColumn operation.

                        static const Word _AESSInvMixColumnTable[4][256].

                                _AESInvMixColumnTable[i][j] contains the contribution of byte
                                j to element i of the InvMixColumn operation.

                                The four bytes of the word _AESInvMixColumnTable[0][j] are:

                                        {0xe}*{j}, {0x9}*{j}, {0xd}*{j}, {0xb}*{j},

                                listed in increasing address order, where multiplication is
                                performed in the Galois field.  {j} designates the element of
                                the Galois field represented by j.  _AESInvMixColumn[i][j] has
                                the same bytes, rotated right in the order shown above.

                        static const Byte _AESRcon[].

                                Round constants, beginning with AESRcon[1] for the first round
                                (AESRcon[0] is padding.)
        
                Arguments:

                        const uint8_t *Key

                                Address of user's cipher key.

                        int Length

                                Number of bytes (16, 24, or 32) or bits (128, 192, or 256) in
                                user's cipher key.

                                This argument is used with _aes_decrypt_key.  It is not
                                present for the other routines.  In those routines, Context
                                is the second argument.

                        aes_decrypt_ctx *Context

                                Structure to contain the expanded key beginning at offset
                                ContextKey and a four-byte "key length" beginning at offset
                                ContextKeyLength.  The "key length" is the number of bytes from
                                the start of the first round key to the startof the last rond
                                key.  That is 16 less than the number of bytes in the entire
                                key.

        Output:

                The expanded key and the "key length" are written to *Context.

        Return:

                aes_rval        // -1 if "key length" is invalid.  0 otherwise.
*/
/* add AES HW detection and program branch if AES HW is detected cclee 3-12-10 */

#ifdef KERNEL
#include <i386/cpu_capabilities.h>
#else
#include <System/i386/cpu_capabilities.h>
#endif

#define dr              r0d                             // Dissection register.
#define drl             r0l                             // Low 8 bits of dissection register.
#define drh             r0h                             // Second-lowest 8 bits of dissection register.

#define t0              r1
#define t0d             r1d                             // Low 32 bits of t0.

#define STable  r2                              // Address of SubBytes table.  Overlaps Nk.
#define ITable  r3                              // Address of InvMixColumn table.
#define offset  Arch(r5, r11)   // Address offset and loop sentinel.

#define R               r7                              // Address of round constant.
#define K               r7                              // User key pointer.
        // R and K overlap.

#define E               r6                              // Expanded key pointer.

#define ve0             %xmm0
#define ve1             %xmm1
#define ve2             %xmm2
#define ve3             %xmm3
#define ve4             %xmm4
#define ve5             %xmm5
#define vt1             %xmm6
#define vt0             %xmm7

#define LookupS(table, index)   (table)*TableSize(STable, index, 4)
#define LookupI(table, index)   (table)*TableSize(ITable, index, 4)


/*      InvMixColumn puts InvMixColumn(dr) into vt0.  This is a non-standard
        subroutine.  It does not conform to the ABI.  It is an integral part of
        _ExpandKeyForDecryption and shares register use with it.
*/
InvMixColumn:
        movzx   drl, t0
        movd    LookupI(0, t0), vt0             // Look up byte 0 in table 0.
        movzx   drh, t0d
        movd    LookupI(1, t0), vt1             // Look up byte 1 in table 1.
        pxor    vt1, vt0
        shr             $16, dr
        movzx   drl, t0d
        movd    LookupI(2, t0), vt1             // Look up byte 2 in table 2.
        pxor    vt1, vt0
        movzx   drh, t0d
        movd    LookupI(3, t0), vt1             // Look up byte 3 in table 3.
        pxor    vt1, vt0
        ret


        // SubWordRotWord adds (XORs) SubWord(RotWord(dr)) to vt0.
        .macro  SubWordRotWord
                movzx   drl, t0
                movd    LookupS(3, t0), vt1             // Look up byte 0 in table 3.
                pxor    vt1, vt0
                movzx   drh, t0d
                movd    LookupS(0, t0), vt1             // Look up byte 1 in table 0.
                pxor    vt1, vt0
                shr             $$16, dr
                movzx   drl, t0d
                movd    LookupS(1, t0), vt1             // Look up byte 2 in table 1.
                pxor    vt1, vt0
                movzx   drh, t0d
                movd    LookupS(2, t0), vt1             // Look up byte 3 in table 2.
                pxor    vt1, vt0
        .endmacro


        // SubWord puts SubWord(dr) into vt0.
        .macro  SubWord
                movzx   drl, t0
                movd    LookupS(0, t0), vt0             // Look up byte 0 in table 0.
                movzx   drh, t0d
                movd    LookupS(1, t0), vt1             // Look up byte 1 in table 1.
                pxor    vt1,vt0
                shr             $$16, dr
                movzx   drl, t0d
                movd    LookupS(2, t0), vt1             // Look up byte 2 in table 2.
                pxor    vt1,vt0
                movzx   drh, t0d
                movd    LookupS(3, t0), vt1             // Look up byte 3 in table 3.
                pxor    vt1,vt0
        .endmacro

        .text
        .globl _aes_decrypt_key
//      .private_extern _aes_decrypt_key
_aes_decrypt_key:

        // detect AES HW, cclee 3-13-10
#if defined __x86_64__
    movq    __cpu_capabilities@GOTPCREL(%rip), %rax                             // %rax -> __cpu_capabilities
    mov     (%rax), %eax                                                                                // %eax  = __cpu_capabilities
#else
#if defined     KERNEL
    leal    __cpu_capabilities, %eax                                                    // %eax -> __cpu_capabilities
    mov     (%eax), %eax                                                                                // %eax  = __cpu_capabilities
#else
        mov    _COMM_PAGE_CPU_CAPABILITIES, %eax
#endif

#endif
    test    $(kHasAES), %eax                                                                    // __cpu_capabilities & kHasAES
    jne     _aes_decrypt_key_hw                                                                 // if AES HW detected, branch to _aes_decrypt_key_hw
        /*      Save registers and set SaveSize to the number of bytes pushed onto the
                stack so far, including the caller's return address.
        */
        push    r3
        #if defined __i386__
                push    r5
                push    r6
                push    r7
                #define SaveSize        (5*4)
        #else
                #define SaveSize        (2*8)
        #endif

        /*      Number of bytes used for local variables:

                        8 16-byte spaces to save XMM registers.

                        8 four-byte spaces for work.
        */
        #define LocalsSize      (8*16 + 8*4)

        // Define stack offset to storage space for local data.
        #define Local   (8*16)

        #if 0 < LocalsSize
                // Padding to position stack pointer at a multiple of 16 bytes.
                #define Padding (15 & -(SaveSize + LocalsSize))
                sub             $Padding + LocalsSize, r4       // Allocate space on stack.
        #else
                #define Padding 0
        #endif

        /*      StackFrame is the number of bytes in our stack frame, from caller's
                stack pointer to ours (so it includes the return address).
        */
        #define StackFrame      (SaveSize + Padding + LocalsSize)

        // Save xmm registers.
        movaps  %xmm0, 0*16(r4)
        movaps  %xmm1, 1*16(r4)
        movaps  %xmm2, 2*16(r4)
        movaps  %xmm3, 3*16(r4)
        movaps  %xmm4, 4*16(r4)
        movaps  %xmm5, 5*16(r4)
        movaps  %xmm6, 6*16(r4)
        movaps  %xmm7, 7*16(r4)

#if defined __i386__

        // Define location of argument i.
        #define Argument(i)     StackFrame+4*(i)(r4)

        #define Nk              t0d

        // Load arguments.
        mov             Argument(2), E
        mov             Argument(1), Nk
        mov             Argument(0), K

#elif defined __x86_64__

        #define Nk              r9d                     // Number of words in key.
        mov             r6d, Nk                         // Move Nk argument out of way.
        mov             r2, E                           // Move E argument to common register.

#endif

        // Dispatch on key length.
        cmp             $128, Nk
        jge             2f
        shl             $3, Nk                          // Convert from bytes to bits.
        cmp             $128, Nk
2:
        je              DKeyHas4Words
        cmp             $192, Nk
        je              DKeyHas6Words
        cmp             $256, Nk
        je              DKeyHas8Words
        mov             $-1, r0                         // Return error.
        jmp             9f


        .globl _aes_decrypt_key128
//      .private_extern _aes_decrypt_key128
_aes_decrypt_key128:

        /*      Save registers and set SaveSize to the number of bytes pushed onto the
                stack so far, including the caller's return address.
        */
        push    r3
        #if defined __i386__
                push    r5
                push    r6
                push    r7
                #define SaveSize        (5*4)
        #else
                #define SaveSize        (2*8)
        #endif

        /*      Number of bytes used for local variables:

                        8 16-byte spaces to save XMM registers.

                        8 four-byte spaces for work.
        */
        #define LocalsSize      (8*16 + 8*4)

        // Define stack offset to storage space for local data.
        #define Local   (8*16)

        #if 0 < LocalsSize
                // Padding to position stack pointer at a multiple of 16 bytes.
                #define Padding (15 & -(SaveSize + LocalsSize))
                sub             $Padding + LocalsSize, r4       // Allocate space on stack.
        #else
                #define Padding 0
        #endif

        /*      StackFrame is the number of bytes in our stack frame, from caller's
                stack pointer to ours (so it includes the return address).
        */
        #define StackFrame      (SaveSize + Padding + LocalsSize)

        // Save xmm registers.
        movaps  %xmm0, 0*16(r4)
        movaps  %xmm1, 1*16(r4)
        movaps  %xmm2, 2*16(r4)
        movaps  %xmm3, 3*16(r4)
        movaps  %xmm4, 4*16(r4)
        movaps  %xmm5, 5*16(r4)
        movaps  %xmm6, 6*16(r4)
        movaps  %xmm7, 7*16(r4)

#if defined __i386__

        // Load arguments.
        #define Argument(i)     StackFrame+4*(i)(r4)
        mov             Argument(1), E
        mov             Argument(0), K

#endif

// Merge point for _aes_decrypt_key and _aes_decrypt_key128.
DKeyHas4Words:

        // First words of expanded key are copied from user key.
        movd    0*4(K), ve0
        movd    1*4(K), ve1
        movd    2*4(K), ve2
        movd    3*4(K), ve3

        movl    $10*16, ContextKeyLength(E)     // Set "key length."

        #if 0 != ContextKey
                add             $ContextKey, E
        #endif

        // K cannot be used after we write to R, since they use the same register.

        #if defined __i386__

                lea             _AESRcon, R
                lea             _AESInvMixColumnTable, ITable
                lea             _AESSubBytesWordTable, STable

        #elif defined __x86_64__

                lea             _AESRcon(%rip), R
                lea             _AESInvMixColumnTable(%rip), ITable
                lea             _AESSubBytesWordTable(%rip), STable

        #endif

        /*      With a four-word key, there are ten rounds (eleven 16-byte key blocks),
                nine of which have InvMixColumn applied.
        */
        mov             $-9*4*4, offset
        sub             offset, E

        // Store initial words of expanded key, which are copies of user's key.
        movd    ve0, 0*4(E, offset)
        movd    ve1, 1*4(E, offset)
        movd    ve2, 2*4(E, offset)
        movd    ve3, 3*4(E, offset)

/*      Here is the first iteration of the key expansion.  It is separate from the
        main loop below because we need to apply InvMixColumn to each of the
        outputs, in ve0 through ve3.  In the main loop, the technique described at
        the top of this file is used to compute the proper outputs while using
        InvMixColumn only once.
*/
        add             $1, R                                   // Advance pointer.
        movd    ve3, dr                                 // Put previous word into work register.
        movzx   (R), t0d                                // Get round constant.
        movd    t0d, vt0

        SubWordRotWord
        pxor    vt0, ve0

        // Chain to successive words.
        pxor    ve0, ve1
        pxor    ve1, ve2
        pxor    ve2, ve3

        add             $4*4, offset

        /*      Apply InvMixColumn to each word.  The transformed values are stored in
                the expanded key.  The original values are retained in registers for
                further computation.
        */
        movd    ve0, dr
        call    InvMixColumn
        movd    vt0, 0*4(E, offset)

        movd    ve1, dr
        call    InvMixColumn
        movd    vt0, 1*4(E, offset)

        movd    ve2, dr
        call    InvMixColumn
        movd    vt0, 2*4(E, offset)

        movd    ve3, dr
        call    InvMixColumn
        movd    vt0, 3*4(E, offset)

//      Here is the main loop.
1:
        add             $1, R                                   // Advance pointer.
        movd    ve3, dr                                 // Put previous word into work register.
        movzx   (R), t0d                                // Get round constant.
        movd    t0d, vt0

        SubWordRotWord
        pxor    vt0, ve0

        // Chain to successive words.
        pxor    ve0, ve1
        pxor    ve1, ve2
        pxor    ve2, ve3
                /*      Dr. Brian Gladman uses a technique with a single XOR here instead
                        of the previous four.  There is some periodic behavior in the key
                        expansion, and Gladman maintains E[4*i+3] for the latest four
                        values of i.  XORing the value in vt0 with one of these yields its
                        replacement.  However, using this technique requires additional
                        instructions before the loop (to initialize the values) and after
                        it (to extract the final values to be stored) and either some way
                        to rotate or index four values in the loop or a four-fold unrolling
                        of the loop to provide the indexing.  Experiment suggests the
                        former is not worthwhile.  Unrolling the loop might give a small
                        gain, at the cost of increased use of instruction cache, increased
                        instructions loads the first time the routine is executed, and
                        increased code complexity, so I decided against it.
                */

        // Apply InvMixColumn to the difference.
        movd    vt0, dr
        call    InvMixColumn

        add             $4*4, offset

        // Chain the transformed difference to previously transformed outputs.
        movd    (0-4)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 0*4(E, offset)

        movd    (1-4)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 1*4(E, offset)

        movd    (2-4)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 2*4(E, offset)

        movd    (3-4)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 3*4(E, offset)

        jl              1b

// Here is the final iteration, which does not perform InvMixColumn.

        movd    ve3, dr                                 // Put previous word into work register.
        movzx   1(R), t0d                               // Get round constant.
        movd    t0d, vt0

        SubWordRotWord
        pxor    vt0, ve0

        // Chain to successive words.
        movd    ve0, 4*4(E, offset)
        pxor    ve0, ve1
        movd    ve1, 5*4(E, offset)
        pxor    ve1, ve2
        movd    ve2, 6*4(E, offset)
        pxor    ve2, ve3
        movd    ve3, 7*4(E, offset)

        xor             r0, r0                          // Return success.

9:
        // Pop stack and restore registers.
        movaps  7*16(r4), %xmm7
        movaps  6*16(r4), %xmm6
        movaps  5*16(r4), %xmm5
        movaps  4*16(r4), %xmm4
        movaps  3*16(r4), %xmm3
        movaps  2*16(r4), %xmm2
        movaps  1*16(r4), %xmm1
        movaps  0*16(r4), %xmm0
        #if 0 < LocalsSize
                add             $Padding + LocalsSize, r4
        #endif
        #if defined __i386__
                pop             r7
                pop             r6
                pop             r5
        #endif
        pop             r3

        ret


        .globl _aes_decrypt_key192
//      .private_extern _aes_decrypt_key192
_aes_decrypt_key192:

        /*      Save registers and set SaveSize to the number of bytes pushed onto the
                stack so far, including the caller's return address.
        */
        push    r3
        #if defined __i386__
                push    r5
                push    r6
                push    r7
                #define SaveSize        (5*4)
        #else
                #define SaveSize        (2*8)
        #endif

        /*      Number of bytes used for local variables:

                        8 16-byte spaces to save XMM registers.

                        8 four-byte spaces for work.
        */
        #define LocalsSize      (8*16 + 8*4)

        // Define stack offset to storage space for local data.
        #define Local   (8*16)

        #if 0 < LocalsSize
                // Padding to position stack pointer at a multiple of 16 bytes.
                #define Padding (15 & -(SaveSize + LocalsSize))
                sub             $Padding + LocalsSize, r4       // Allocate space on stack.
        #else
                #define Padding 0
        #endif

        /*      StackFrame is the number of bytes in our stack frame, from caller's
                stack pointer to ours (so it includes the return address).
        */
        #define StackFrame      (SaveSize + Padding + LocalsSize)

        // Save xmm registers.
        movaps  %xmm0, 0*16(r4)
        movaps  %xmm1, 1*16(r4)
        movaps  %xmm2, 2*16(r4)
        movaps  %xmm3, 3*16(r4)
        movaps  %xmm4, 4*16(r4)
        movaps  %xmm5, 5*16(r4)
        movaps  %xmm6, 6*16(r4)
        movaps  %xmm7, 7*16(r4)

#if defined __i386__

        // Load arguments.
        #define Argument(i)     StackFrame+4*(i)(r4)
        mov             Argument(1), E
        mov             Argument(0), K

#endif

// Merge point for _aes_decrypt_key and _aes_decrypt_key192.
DKeyHas6Words:

        // First words of expanded key are copied from user key.
        movd    0*4(K), ve0
        movd    1*4(K), ve1
        movd    2*4(K), ve2
        movd    3*4(K), ve3

        movl    $12*16, ContextKeyLength(E)     // Set "key length."

        #if 0 != ContextKey
                add             $ContextKey, E
        #endif

        movd    4*4(K), ve4
        movd    5*4(K), ve5

        // K cannot be used after we write to R, since they use the same register.

        #if defined __i386__

                lea             _AESRcon, R
                lea             _AESInvMixColumnTable, ITable
                lea             _AESSubBytesWordTable, STable

        #elif defined __x86_64__

                lea             _AESRcon(%rip), R
                lea             _AESInvMixColumnTable(%rip), ITable
                lea             _AESSubBytesWordTable(%rip), STable

        #endif

        /*      With a six-word key, there are twelve rounds (thirteen 16-byte key
                blocks), eleven of which have InvMixColumn applied.  The key expansion
                proceeds in iterations of six four-byte words, so the termination
                condition is a bit complicated.  We set offset to the negative of 10
                four four-byte words, and the loop branch does another iteration if
                offset is less than or equal to zero, meaning the number of iterations
                performed so far is less than or equal to 10.  Thus, after ten
                iterations, it branches again.  After the eleventh iteration, it
                stops.  Code after the end of the loop computes the twelfth key block,
                which does not have InvMixColumn applied.
        */
        mov             $-10*4*4, offset
        sub             offset, E

        // Store initial words of expanded key, which are copies of user's key.
        movd    ve0, 0*4(E, offset)
        movd    ve1, 1*4(E, offset)
        movd    ve2, 2*4(E, offset)
        movd    ve3, 3*4(E, offset)

        /*      The first four words are stored untransformed.  After that, words in
                the expanded key are transformed by InvMixColumn.
        */
        movd    ve4, dr
        call    InvMixColumn
        movd    vt0, 4*4(E, offset)

        movd    ve5, dr
        call    InvMixColumn
        movd    vt0, 5*4(E, offset)

/*      Here is the first iteration of the key expansion.  It is separate from the
        main loop below because we need to apply InvMixColumn to each of the
        outputs, in ve0 through ve5.  In the main loop, the technique described at
        the top of this file is used to compute the proper outputs while using
        InvMixColumn only once.
*/
        add             $1, R                                   // Advance pointer.
        movd    ve5, dr                                 // Put previous word into work register.
        movzx   (R), t0d                                // Get round constant.
        movd    t0d, vt0

        SubWordRotWord
        pxor    vt0, ve0

        // Chain to successive words.
        pxor    ve0, ve1
        pxor    ve1, ve2
        pxor    ve2, ve3
        pxor    ve3, ve4
        pxor    ve4, ve5

        add             $6*4, offset

        /*      Apply InvMixColumn to each word.  The transformed values are stored in
                the expanded key.  The original values are retained in registers for
                further computation.
        */
        movd    ve0, dr
        call    InvMixColumn
        movd    vt0, 0*4(E, offset)

        movd    ve1, dr
        call    InvMixColumn
        movd    vt0, 1*4(E, offset)

        movd    ve2, dr
        call    InvMixColumn
        movd    vt0, 2*4(E, offset)

        movd    ve3, dr
        call    InvMixColumn
        movd    vt0, 3*4(E, offset)

        movd    (4-6)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 4*4(E, offset)

        movd    (5-6)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 5*4(E, offset)

//      Here is the main loop.
1:
        add             $1, R                                   // Advance pointer.
        movd    ve5, dr                                 // Put previous word into work register.
        movzx   (R), t0d                                // Get round constant.
        movd    t0d, vt0

        SubWordRotWord
        pxor    vt0, ve0

        // Chain to successive words.
        pxor    ve0, ve1
        pxor    ve1, ve2
        pxor    ve2, ve3
        pxor    ve3, ve4
        pxor    ve4, ve5

        // Apply InvMixColumn to the difference.
        movd    vt0, dr
        call    InvMixColumn

        add             $6*4, offset

        // Chain the transformed difference to previously transformed outputs.
        movd    (0-6)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 0*4(E, offset)

        movd    (1-6)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 1*4(E, offset)

        movd    (2-6)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 2*4(E, offset)

        movd    (3-6)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 3*4(E, offset)

        movd    (4-6)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 4*4(E, offset)

        movd    (5-6)*4(E, offset), vt1
        pxor    vt1, vt0
        movd    vt0, 5*4(E, offset)

        jle             1b

// Here is the final iteration, which does not perform InvMixColumn.

        movd    ve5, dr                                 // Put previous word into work register.
        movzx   1(R), t0d                               // Get round constant.
        movd    t0d, vt0

        SubWordRotWord
        pxor    vt0, ve0

        // Chain to successive words.
        movd    ve0, 6*4(E, offset)
        pxor    ve0, ve1
        movd    ve1, 7*4(E, offset)
        pxor    ve1, ve2
        movd    ve2, 8*4(E, offset)
        pxor    ve2, ve3
        movd    ve3, 9*4(E, offset)

        xor             r0, r0                          // Return success.

        // Pop stack and restore registers.
        movaps  7*16(r4), %xmm7
        movaps  6*16(r4), %xmm6
        movaps  5*16(r4), %xmm5
        movaps  4*16(r4), %xmm4
        movaps  3*16(r4), %xmm3
        movaps  2*16(r4), %xmm2
        movaps  1*16(r4), %xmm1
        movaps  0*16(r4), %xmm0
        #if 0 < LocalsSize
                add             $Padding + LocalsSize, r4
        #endif
        #if defined __i386__
                pop             r7
                pop             r6
                pop             r5
        #endif
        pop             r3

        ret


        .globl _aes_decrypt_key256
//      .private_extern _aes_decrypt_key256
_aes_decrypt_key256:

        /*      Save registers and set SaveSize to the number of bytes pushed onto the
                stack so far, including the caller's return address.
        */
        push    r3
        #if defined __i386__
                push    r5
                push    r6
                push    r7
                #define SaveSize        (5*4)
        #else
                #define SaveSize        (2*8)
        #endif

        /*      Number of bytes used for local variables:

                        8 16-byte spaces to save XMM registers.

                        8 four-byte spaces for work.
        */
        #define LocalsSize      (8*16 + 8*4)

        // Define stack offset to storage space for local data.
        #define Local   (8*16)

        #if 0 < LocalsSize
                // Padding to position stack pointer at a multiple of 16 bytes.
                #define Padding (15 & -(SaveSize + LocalsSize))
                sub             $Padding + LocalsSize, r4       // Allocate space on stack.
        #else
                #define Padding 0
        #endif

        /*      StackFrame is the number of bytes in our stack frame, from caller's
                stack pointer to ours (so it includes the return address).
        */
        #define StackFrame      (SaveSize + Padding + LocalsSize)

        // Save xmm registers.
        movaps  %xmm0, 0*16(r4)
        movaps  %xmm1, 1*16(r4)
        movaps  %xmm2, 2*16(r4)
        movaps  %xmm3, 3*16(r4)
        movaps  %xmm4, 4*16(r4)
        movaps  %xmm5, 5*16(r4)
        movaps  %xmm6, 6*16(r4)
        movaps  %xmm7, 7*16(r4)

#if defined __i386__

        // Load arguments.
        #define Argument(i)     StackFrame+4*(i)(r4)
        mov             Argument(1), E
        mov             Argument(0), K

#endif

// Merge point for _aes_decrypt_key and _aes_decrypt_key256.
DKeyHas8Words:

        // First words of expanded key are copied from user key.
        movd    0*4(K), ve0
        movd    1*4(K), ve1
        movd    2*4(K), ve2
        movd    3*4(K), ve3

        movl    $14*16, ContextKeyLength(E)     // Set "key length."

        #if 0 != ContextKey
                add             $ContextKey, E
        #endif

        // Store initial words of expanded key, which are copies of user's key.
        movd    ve0, 0*4(E)
        movd    ve1, 1*4(E)
        movd    ve2, 2*4(E)
        movd    ve3, 3*4(E)
        movd    4*4(K), ve0
        movd    5*4(K), ve1
        movd    6*4(K), ve2
        movd    7*4(K), ve3

        // K cannot be used after we write to R, since they use the same register.

        #if defined __i386__

                lea             _AESRcon, R
                lea             _AESInvMixColumnTable, ITable
                lea             _AESSubBytesWordTable, STable

        #elif defined __x86_64__

                lea             _AESRcon(%rip), R
                lea             _AESInvMixColumnTable(%rip), ITable
                lea             _AESSubBytesWordTable(%rip), STable

        #endif

        /*      With an eight-word key, there are fourteen rounds (fifteen 16-byte key
                blocks), thirteen of which have InvMixColumn applied.
        */
        mov             $-12*4*4, offset
        sub             offset, E

        // Save untransformed values in stack area.
        movd    ve0, 4*4+Local(r4)
        movd    ve1, 5*4+Local(r4)
        movd    ve2, 6*4+Local(r4)
        movd    ve3, 7*4+Local(r4)

        /*      Apply InvMixColumn to words 4 through 7.  The transformed values are
                stored in the expanded key.  The original values are saved in the stack
                area for further computation.
        */
        movd    ve0, dr
        call    InvMixColumn
        movd    vt0, 4*4(E, offset)

        movd    ve1, dr
        call    InvMixColumn
        movd    vt0, 5*4(E, offset)

        movd    ve2, dr
        call    InvMixColumn
        movd    vt0, 6*4(E, offset)

        movd    ve3, dr
        call    InvMixColumn
        movd    vt0, 7*4(E, offset)

/*      Here is the first iteration of the key expansion.  It is separate from the
        main loop below because we need to apply InvMixColumn to each of the
        outputs, in ve0 through ve3.  In the main loop, the technique described at
        the top of this file is used to compute the proper outputs while using
        InvMixColumn only once.
*/
        add             $1, R                                   // Advance pointer.
        movd    ve3, dr                                 // Put previous word into work register.
        movzx   (R), t0d                                // Get round constant.
        movd    t0d, vt0

        SubWordRotWord

        add             $8*4, offset

        movd    (0-8)*4(E, offset), ve0         // Get old word.
        pxor    vt0, ve0
        movd    ve0, 0*4+Local(r4)                      // Save on stack.
        movd    ve0, dr
        call    InvMixColumn
        movd    vt0, 0*4(E, offset)                     // Write to expanded key.

        /*       Chain to successive words and apply InvMixColumn to each word.  The
                 transformed values are stored in the expanded key.  The original
                 values are retained in local data for further computation.
        */
        movd    (1-8)*4(E, offset), ve1         // Get old word.
        pxor    ve0, ve1                                        // Chain.
        movd    ve1, 1*4+Local(r4)                      // Save on stack.
        movd    ve1, dr
        call    InvMixColumn
        movd    vt0, 1*4(E, offset)                     // Write to expanded key.

        movd    (2-8)*4(E, offset), ve2         // Get old word.
        pxor    ve1, ve2                                        // Chain.
        movd    ve2, 2*4+Local(r4)                      // Save on stack.
        movd    ve2, dr
        call    InvMixColumn
        movd    vt0, 2*4(E, offset)                     // Write to expanded key.

        movd    (3-8)*4(E, offset), ve3         // Get old word.
        pxor    ve2, ve3                                        // Chain.
        movd    ve3, 3*4+Local(r4)                      // Save on stack.
        movd    ve3, dr
        call    InvMixColumn
        movd    vt0, 3*4(E, offset)                     // Write to expanded key.

        movd    ve3, dr                                         // Put previous word into work register.
        SubWord

        movd    4*4+Local(r4), ve0                      // Get old word.
        pxor    vt0, ve0                                        // Chain.
        movd    ve0, 4*4+Local(r4)                      // Save on stack.

        movd    5*4+Local(r4), ve1                      // Get old word.
        pxor    ve0, ve1                                        // Chain.
        movd    ve1, 5*4+Local(r4)                      // Save on stack.

        movd    6*4+Local(r4), ve2                      // Get old word.
        pxor    ve1, ve2                                        // Chain.
        movd    ve2, 6*4+Local(r4)                      // Save on stack.

        movd    7*4+Local(r4), ve3                      // Get old word.
        pxor    ve2, ve3                                        // Chain.
        movd    ve3, 7*4+Local(r4)                      // Save on stack.

        movd    vt0, dr                                         // Move change to work register.
        call    InvMixColumn

        movd    (4-8)*4(E, offset), vt1         // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, 4*4(E, offset)                     // Write new word to expanded key.

        movd    (5-8)*4(E, offset), vt1         // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, 5*4(E, offset)                     // Write new word to expanded key.

        movd    (6-8)*4(E, offset), vt1         // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, 6*4(E, offset)                     // Write new word to expanded key.

        movd    (7-8)*4(E, offset), vt1         // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, 7*4(E, offset)                     // Write new word to expanded key.

//      Here is the main loop.
1:
        add             $1, R                                           // Advance pointer.
        movd    ve3, dr                                         // Put previous word into work register.
        movzx   (R), t0d                                        // Get round constant.
        movd    t0d, vt0

        SubWordRotWord

        movd    0*4+Local(r4), ve0                      // Get old word.
        pxor    vt0, ve0
        movd    ve0, 0*4+Local(r4)                      // Save on stack.

        // Chain to successive words.
        movd    1*4+Local(r4), ve1                      // Get old word.
        pxor    ve0, ve1                                        // Chain.
        movd    ve1, 1*4+Local(r4)                      // Save on stack.

        movd    2*4+Local(r4), ve2                      // Get old word.
        pxor    ve1, ve2                                        // Chain.
        movd    ve2, 2*4+Local(r4)                      // Save on stack.

        movd    3*4+Local(r4), ve3                      // Get old word.
        pxor    ve2, ve3                                        // Chain.
        movd    ve3, 3*4+Local(r4)                      // Save on stack.

        movd    vt0, dr                                         // Move change to work register.
        call    InvMixColumn

        movd    0*4(E, offset), vt1                     // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, (0+8)*4(E, offset)         // Write new word to expanded key.

        movd    1*4(E, offset), vt1                     // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, (1+8)*4(E, offset)         // Write new word to expanded key.

        movd    2*4(E, offset), vt1                     // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, (2+8)*4(E, offset)         // Write new word to expanded key.

        movd    3*4(E, offset), vt1                     // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, (3+8)*4(E, offset)         // Write new word to expanded key.

        movd    ve3, dr                                         // Put previous word into work register.
        SubWord

        movd    4*4+Local(r4), ve0                      // Get old word.
        pxor    vt0, ve0                                        // Chain.
        movd    ve0, 4*4+Local(r4)                      // Save on stack.

        movd    5*4+Local(r4), ve1                      // Get old word.
        pxor    ve0, ve1                                        // Chain.
        movd    ve1, 5*4+Local(r4)                      // Save on stack.

        movd    6*4+Local(r4), ve2                      // Get old word.
        pxor    ve1, ve2                                        // Chain.
        movd    ve2, 6*4+Local(r4)                      // Save on stack.

        movd    7*4+Local(r4), ve3                      // Get old word.
        pxor    ve2, ve3                                        // Chain.
        movd    ve3, 7*4+Local(r4)                      // Save on stack.

        movd    vt0, dr                                         // Move change to work register.
        call    InvMixColumn

        movd    4*4(E, offset), vt1                     // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, (4+8)*4(E, offset)         // Write new word to expanded key.

        movd    5*4(E, offset), vt1                     // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, (5+8)*4(E, offset)         // Write new word to expanded key.

        movd    6*4(E, offset), vt1                     // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, (6+8)*4(E, offset)         // Write new word to expanded key.

        movd    7*4(E, offset), vt1                     // Get old word.
        pxor    vt1, vt0                                        // Chain.
        movd    vt0, (7+8)*4(E, offset)         // Write new word to expanded key.

        add             $8*4, offset

        jl              1b

        movd    ve3, dr                                         // Put previous word into work register.
        movzx   1(R), t0d                                       // Get round constant.
        movd    t0d, vt0

        SubWordRotWord

        movd    0*4+Local(r4), ve0                      // Get old word.
        pxor    vt0, ve0                                        // Chain.
        movd    ve0, (0+8)*4(E, offset)

        // Chain to successive words.
        movd    1*4+Local(r4), ve1                      // Get old word.
        pxor    ve0, ve1                                        // Chain.
        movd    ve1, (1+8)*4(E, offset)

        movd    2*4+Local(r4), ve2                      // Get old word.
        pxor    ve1, ve2                                        // Chain.
        movd    ve2, (2+8)*4(E, offset)

        movd    3*4+Local(r4), ve3                      // Get old word.
        pxor    ve2, ve3                                        // Chain.
        movd    ve3, (3+8)*4(E, offset)

        xor             r0, r0                          // Return success.

        // Pop stack and restore registers.
        movaps  7*16(r4), %xmm7
        movaps  6*16(r4), %xmm6
        movaps  5*16(r4), %xmm5
        movaps  4*16(r4), %xmm4
        movaps  3*16(r4), %xmm3
        movaps  2*16(r4), %xmm2
        movaps  1*16(r4), %xmm1
        movaps  0*16(r4), %xmm0
        #if 0 < LocalsSize
                add             $Padding + LocalsSize, r4
        #endif
        #if defined __i386__
                pop             r7
                pop             r6
                pop             r5
        #endif
        pop             r3

        ret


#undef  Address
#undef  Argument
#undef  E
#undef  ITable
#undef  K
#undef  Local
#undef  LocalsSize
#undef  LookupI
#undef  LookupS
#undef  Nk
#undef  Padding
#undef  R
#undef  SaveSize
#undef  STable
#undef  StackFrame
#undef  dr
#undef  drh
#undef  drl
#undef  offset
#undef  t0
#undef  t0d
#undef  ve0
#undef  ve1
#undef  ve2
#undef  ve3
#undef  ve4
#undef  ve5
#undef  vt0
#undef  vt1