--- /dev/null
+/* This file defines _aes_decrypt_key, _aes_decrypt_key128,
+ _aes_decrypt_key192, and _aes_decrypt_key256. It is designed to be
+ included in another assembly file with the preprocessor #include directive,
+ to benefit from some assembly-time calculations.
+
+ Written by Eric Postpischil, January 2008.
+
+ The comments here do not say much about the algorithm; the code just
+ follows the FIPS-197 specification. I recommend reading the specification
+ before working with this code or examining the C code in the parent
+ directory that illustrates key expansion.
+
+ One complication is that this routine both expands the key and applies
+ InvMixColumn to most of the words in the expanded key. This modifies the
+ key for use with the Equivalent Inverse Cipher.
+
+ During key expansion, there are sequences of four or six words that are
+ produced like this:
+
+ E[i+0] = E[i+0-Nk] ^ f(E[i-1]), where f is some function.
+ E[i+1] = E[i+1-Nk] ^ E[i+0].
+ E[i+2] = E[i+2-Nk] ^ E[i+1].
+ E[i+3] = E[i+3-Nk] ^ E[i+2].
+
+ When Nk is four or eight, the sequence stops there. When it is six, it
+ goes on for two more words. Let I be the InvMixColumn function. for the
+ Equivalent Inverse Cipher, we want to store I(E[i+0]), I(E[i+1]),
+ I(E[i+2]), I(E[i+3]) (and two more when Nk is six). However, we do not
+ need to calculate I four times. In AES' finite field, I is a linear
+ combination of the four bytes of its input. The ^ operation on the bits
+ that represent field elements is an addition in the Galois field. So
+ I(a ^ b) = I(a) ^ I(b). Then we have:
+
+ I(E[i+0]) = I(E[i+0-Nk] ^ f(E[i-1])) = I(E[i+0-Nk]) ^ I(f(E[i-1])).
+ I(E[i+1]) = I(E[i+1-Nk]) ^ I(E[i+0]).
+ I(E[i+2]) = I(E[i+2-Nk]) ^ I(E[i+1]).
+ I(E[i+3]) = I(E[i+3-Nk]) ^ I(E[i+2]).
+
+ To compute this, we compute I(f(E[i-1])) and XOR it with the previously
+ stored E[i+0-Nk])) to get I(E[i+0])). Then we XOR that with the previously
+ stored E[i+1-Nk])) to get I(E[i+1])), and so on.
+
+ Note that to compute I(f(E[i-1])), we need to have E[i-1]. So we have to
+ compute the pre-InvMixColumn words of the expanded key; it is not
+ sufficient to have the post-InvMixColumn words.
+*/
+
+
+/* Routine:
+
+ _aes_decrypt_key.
+
+ _aes_decrypt_key128, _aes_decrypt_key192, and _aes_decrypt_key256.
+
+ Function:
+
+ Expand the user's cipher key into the key schedule, as defined in
+ Federal Information Processing Standards Publication 197 (FIPS-197),
+ November 26, 2001.
+
+ For decryption, the key is modified as shown in Figure 15 in FIPS-197,
+ to support the Equivalent Inverse Cipher.
+
+ Input:
+
+ Constant data:
+
+ The following names must be locally defined so the assembler
+ can calculate certain offsets.
+
+ static const Word _AESSubBytesWordTable[4][256].
+
+ _AESSubBytesWordTable[i][j] = SubBytes(j) << 8*i, where
+ SubBytes is defined in FIPS-197. _AESSubBytesWordTable
+ differs from _AESEncryptTable in that it does not include
+ the MixColumn operation. It is used in performing the last
+ round, which differs fromm the previous rounds in that it
+ does not include the MixColumn operation.
+
+ static const Word _AESSInvMixColumnTable[4][256].
+
+ _AESInvMixColumnTable[i][j] contains the contribution of byte
+ j to element i of the InvMixColumn operation.
+
+ The four bytes of the word _AESInvMixColumnTable[0][j] are:
+
+ {0xe}*{j}, {0x9}*{j}, {0xd}*{j}, {0xb}*{j},
+
+ listed in increasing address order, where multiplication is
+ performed in the Galois field. {j} designates the element of
+ the Galois field represented by j. _AESInvMixColumn[i][j] has
+ the same bytes, rotated right in the order shown above.
+
+ static const Byte _AESRcon[].
+
+ Round constants, beginning with AESRcon[1] for the first round
+ (AESRcon[0] is padding.)
+
+ Arguments:
+
+ const uint8_t *Key
+
+ Address of user's cipher key.
+
+ int Length
+
+ Number of bytes (16, 24, or 32) or bits (128, 192, or 256) in
+ user's cipher key.
+
+ This argument is used with _aes_decrypt_key. It is not
+ present for the other routines. In those routines, Context
+ is the second argument.
+
+ aes_decrypt_ctx *Context
+
+ Structure to contain the expanded key beginning at offset
+ ContextKey and a four-byte "key length" beginning at offset
+ ContextKeyLength. The "key length" is the number of bytes from
+ the start of the first round key to the startof the last rond
+ key. That is 16 less than the number of bytes in the entire
+ key.
+
+ Output:
+
+ The expanded key and the "key length" are written to *Context.
+
+ Return:
+
+ aes_rval // -1 if "key length" is invalid. 0 otherwise.
+*/
+/* add AES HW detection and program branch if AES HW is detected cclee 3-12-10 */
+
+#ifdef KERNEL
+#include <i386/cpu_capabilities.h>
+#else
+#include <System/i386/cpu_capabilities.h>
+#endif
+
+#define dr r0d // Dissection register.
+#define drl r0l // Low 8 bits of dissection register.
+#define drh r0h // Second-lowest 8 bits of dissection register.
+
+#define t0 r1
+#define t0d r1d // Low 32 bits of t0.
+
+#define STable r2 // Address of SubBytes table. Overlaps Nk.
+#define ITable r3 // Address of InvMixColumn table.
+#define offset Arch(r5, r11) // Address offset and loop sentinel.
+
+#define R r7 // Address of round constant.
+#define K r7 // User key pointer.
+ // R and K overlap.
+
+#define E r6 // Expanded key pointer.
+
+#define ve0 %xmm0
+#define ve1 %xmm1
+#define ve2 %xmm2
+#define ve3 %xmm3
+#define ve4 %xmm4
+#define ve5 %xmm5
+#define vt1 %xmm6
+#define vt0 %xmm7
+
+#define LookupS(table, index) (table)*TableSize(STable, index, 4)
+#define LookupI(table, index) (table)*TableSize(ITable, index, 4)
+
+
+/* InvMixColumn puts InvMixColumn(dr) into vt0. This is a non-standard
+ subroutine. It does not conform to the ABI. It is an integral part of
+ _ExpandKeyForDecryption and shares register use with it.
+*/
+InvMixColumn:
+ movzx drl, t0
+ movd LookupI(0, t0), vt0 // Look up byte 0 in table 0.
+ movzx drh, t0d
+ movd LookupI(1, t0), vt1 // Look up byte 1 in table 1.
+ pxor vt1, vt0
+ shr $16, dr
+ movzx drl, t0d
+ movd LookupI(2, t0), vt1 // Look up byte 2 in table 2.
+ pxor vt1, vt0
+ movzx drh, t0d
+ movd LookupI(3, t0), vt1 // Look up byte 3 in table 3.
+ pxor vt1, vt0
+ ret
+
+
+ // SubWordRotWord adds (XORs) SubWord(RotWord(dr)) to vt0.
+ .macro SubWordRotWord
+ movzx drl, t0
+ movd LookupS(3, t0), vt1 // Look up byte 0 in table 3.
+ pxor vt1, vt0
+ movzx drh, t0d
+ movd LookupS(0, t0), vt1 // Look up byte 1 in table 0.
+ pxor vt1, vt0
+ shr $$16, dr
+ movzx drl, t0d
+ movd LookupS(1, t0), vt1 // Look up byte 2 in table 1.
+ pxor vt1, vt0
+ movzx drh, t0d
+ movd LookupS(2, t0), vt1 // Look up byte 3 in table 2.
+ pxor vt1, vt0
+ .endmacro
+
+
+ // SubWord puts SubWord(dr) into vt0.
+ .macro SubWord
+ movzx drl, t0
+ movd LookupS(0, t0), vt0 // Look up byte 0 in table 0.
+ movzx drh, t0d
+ movd LookupS(1, t0), vt1 // Look up byte 1 in table 1.
+ pxor vt1,vt0
+ shr $$16, dr
+ movzx drl, t0d
+ movd LookupS(2, t0), vt1 // Look up byte 2 in table 2.
+ pxor vt1,vt0
+ movzx drh, t0d
+ movd LookupS(3, t0), vt1 // Look up byte 3 in table 3.
+ pxor vt1,vt0
+ .endmacro
+
+ .text
+ .globl _aes_decrypt_key
+// .private_extern _aes_decrypt_key
+_aes_decrypt_key:
+
+ // detect AES HW, cclee 3-13-10
+#if defined __x86_64__
+ movq __cpu_capabilities@GOTPCREL(%rip), %rax // %rax -> __cpu_capabilities
+ mov (%rax), %eax // %eax = __cpu_capabilities
+#else
+#if defined KERNEL
+ leal __cpu_capabilities, %eax // %eax -> __cpu_capabilities
+ mov (%eax), %eax // %eax = __cpu_capabilities
+#else
+ mov _COMM_PAGE_CPU_CAPABILITIES, %eax
+#endif
+
+#endif
+ test $(kHasAES), %eax // __cpu_capabilities & kHasAES
+ jne _aes_decrypt_key_hw // if AES HW detected, branch to _aes_decrypt_key_hw
+ /* Save registers and set SaveSize to the number of bytes pushed onto the
+ stack so far, including the caller's return address.
+ */
+ push r3
+ #if defined __i386__
+ push r5
+ push r6
+ push r7
+ #define SaveSize (5*4)
+ #else
+ #define SaveSize (2*8)
+ #endif
+
+ /* Number of bytes used for local variables:
+
+ 8 16-byte spaces to save XMM registers.
+
+ 8 four-byte spaces for work.
+ */
+ #define LocalsSize (8*16 + 8*4)
+
+ // Define stack offset to storage space for local data.
+ #define Local (8*16)
+
+ #if 0 < LocalsSize
+ // Padding to position stack pointer at a multiple of 16 bytes.
+ #define Padding (15 & -(SaveSize + LocalsSize))
+ sub $Padding + LocalsSize, r4 // Allocate space on stack.
+ #else
+ #define Padding 0
+ #endif
+
+ /* StackFrame is the number of bytes in our stack frame, from caller's
+ stack pointer to ours (so it includes the return address).
+ */
+ #define StackFrame (SaveSize + Padding + LocalsSize)
+
+ // Save xmm registers.
+ movaps %xmm0, 0*16(r4)
+ movaps %xmm1, 1*16(r4)
+ movaps %xmm2, 2*16(r4)
+ movaps %xmm3, 3*16(r4)
+ movaps %xmm4, 4*16(r4)
+ movaps %xmm5, 5*16(r4)
+ movaps %xmm6, 6*16(r4)
+ movaps %xmm7, 7*16(r4)
+
+#if defined __i386__
+
+ // Define location of argument i.
+ #define Argument(i) StackFrame+4*(i)(r4)
+
+ #define Nk t0d
+
+ // Load arguments.
+ mov Argument(2), E
+ mov Argument(1), Nk
+ mov Argument(0), K
+
+#elif defined __x86_64__
+
+ #define Nk r9d // Number of words in key.
+ mov r6d, Nk // Move Nk argument out of way.
+ mov r2, E // Move E argument to common register.
+
+#endif
+
+ // Dispatch on key length.
+ cmp $128, Nk
+ jge 2f
+ shl $3, Nk // Convert from bytes to bits.
+ cmp $128, Nk
+2:
+ je DKeyHas4Words
+ cmp $192, Nk
+ je DKeyHas6Words
+ cmp $256, Nk
+ je DKeyHas8Words
+ mov $-1, r0 // Return error.
+ jmp 9f
+
+
+ .globl _aes_decrypt_key128
+// .private_extern _aes_decrypt_key128
+_aes_decrypt_key128:
+
+ /* Save registers and set SaveSize to the number of bytes pushed onto the
+ stack so far, including the caller's return address.
+ */
+ push r3
+ #if defined __i386__
+ push r5
+ push r6
+ push r7
+ #define SaveSize (5*4)
+ #else
+ #define SaveSize (2*8)
+ #endif
+
+ /* Number of bytes used for local variables:
+
+ 8 16-byte spaces to save XMM registers.
+
+ 8 four-byte spaces for work.
+ */
+ #define LocalsSize (8*16 + 8*4)
+
+ // Define stack offset to storage space for local data.
+ #define Local (8*16)
+
+ #if 0 < LocalsSize
+ // Padding to position stack pointer at a multiple of 16 bytes.
+ #define Padding (15 & -(SaveSize + LocalsSize))
+ sub $Padding + LocalsSize, r4 // Allocate space on stack.
+ #else
+ #define Padding 0
+ #endif
+
+ /* StackFrame is the number of bytes in our stack frame, from caller's
+ stack pointer to ours (so it includes the return address).
+ */
+ #define StackFrame (SaveSize + Padding + LocalsSize)
+
+ // Save xmm registers.
+ movaps %xmm0, 0*16(r4)
+ movaps %xmm1, 1*16(r4)
+ movaps %xmm2, 2*16(r4)
+ movaps %xmm3, 3*16(r4)
+ movaps %xmm4, 4*16(r4)
+ movaps %xmm5, 5*16(r4)
+ movaps %xmm6, 6*16(r4)
+ movaps %xmm7, 7*16(r4)
+
+#if defined __i386__
+
+ // Load arguments.
+ #define Argument(i) StackFrame+4*(i)(r4)
+ mov Argument(1), E
+ mov Argument(0), K
+
+#endif
+
+// Merge point for _aes_decrypt_key and _aes_decrypt_key128.
+DKeyHas4Words:
+
+ // First words of expanded key are copied from user key.
+ movd 0*4(K), ve0
+ movd 1*4(K), ve1
+ movd 2*4(K), ve2
+ movd 3*4(K), ve3
+
+ movl $10*16, ContextKeyLength(E) // Set "key length."
+
+ #if 0 != ContextKey
+ add $ContextKey, E
+ #endif
+
+ // K cannot be used after we write to R, since they use the same register.
+
+ #if defined __i386__
+
+ lea _AESRcon, R
+ lea _AESInvMixColumnTable, ITable
+ lea _AESSubBytesWordTable, STable
+
+ #elif defined __x86_64__
+
+ lea _AESRcon(%rip), R
+ lea _AESInvMixColumnTable(%rip), ITable
+ lea _AESSubBytesWordTable(%rip), STable
+
+ #endif
+
+ /* With a four-word key, there are ten rounds (eleven 16-byte key blocks),
+ nine of which have InvMixColumn applied.
+ */
+ mov $-9*4*4, offset
+ sub offset, E
+
+ // Store initial words of expanded key, which are copies of user's key.
+ movd ve0, 0*4(E, offset)
+ movd ve1, 1*4(E, offset)
+ movd ve2, 2*4(E, offset)
+ movd ve3, 3*4(E, offset)
+
+/* Here is the first iteration of the key expansion. It is separate from the
+ main loop below because we need to apply InvMixColumn to each of the
+ outputs, in ve0 through ve3. In the main loop, the technique described at
+ the top of this file is used to compute the proper outputs while using
+ InvMixColumn only once.
+*/
+ add $1, R // Advance pointer.
+ movd ve3, dr // Put previous word into work register.
+ movzx (R), t0d // Get round constant.
+ movd t0d, vt0
+
+ SubWordRotWord
+ pxor vt0, ve0
+
+ // Chain to successive words.
+ pxor ve0, ve1
+ pxor ve1, ve2
+ pxor ve2, ve3
+
+ add $4*4, offset
+
+ /* Apply InvMixColumn to each word. The transformed values are stored in
+ the expanded key. The original values are retained in registers for
+ further computation.
+ */
+ movd ve0, dr
+ call InvMixColumn
+ movd vt0, 0*4(E, offset)
+
+ movd ve1, dr
+ call InvMixColumn
+ movd vt0, 1*4(E, offset)
+
+ movd ve2, dr
+ call InvMixColumn
+ movd vt0, 2*4(E, offset)
+
+ movd ve3, dr
+ call InvMixColumn
+ movd vt0, 3*4(E, offset)
+
+// Here is the main loop.
+1:
+ add $1, R // Advance pointer.
+ movd ve3, dr // Put previous word into work register.
+ movzx (R), t0d // Get round constant.
+ movd t0d, vt0
+
+ SubWordRotWord
+ pxor vt0, ve0
+
+ // Chain to successive words.
+ pxor ve0, ve1
+ pxor ve1, ve2
+ pxor ve2, ve3
+ /* Dr. Brian Gladman uses a technique with a single XOR here instead
+ of the previous four. There is some periodic behavior in the key
+ expansion, and Gladman maintains E[4*i+3] for the latest four
+ values of i. XORing the value in vt0 with one of these yields its
+ replacement. However, using this technique requires additional
+ instructions before the loop (to initialize the values) and after
+ it (to extract the final values to be stored) and either some way
+ to rotate or index four values in the loop or a four-fold unrolling
+ of the loop to provide the indexing. Experiment suggests the
+ former is not worthwhile. Unrolling the loop might give a small
+ gain, at the cost of increased use of instruction cache, increased
+ instructions loads the first time the routine is executed, and
+ increased code complexity, so I decided against it.
+ */
+
+ // Apply InvMixColumn to the difference.
+ movd vt0, dr
+ call InvMixColumn
+
+ add $4*4, offset
+
+ // Chain the transformed difference to previously transformed outputs.
+ movd (0-4)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 0*4(E, offset)
+
+ movd (1-4)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 1*4(E, offset)
+
+ movd (2-4)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 2*4(E, offset)
+
+ movd (3-4)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 3*4(E, offset)
+
+ jl 1b
+
+// Here is the final iteration, which does not perform InvMixColumn.
+
+ movd ve3, dr // Put previous word into work register.
+ movzx 1(R), t0d // Get round constant.
+ movd t0d, vt0
+
+ SubWordRotWord
+ pxor vt0, ve0
+
+ // Chain to successive words.
+ movd ve0, 4*4(E, offset)
+ pxor ve0, ve1
+ movd ve1, 5*4(E, offset)
+ pxor ve1, ve2
+ movd ve2, 6*4(E, offset)
+ pxor ve2, ve3
+ movd ve3, 7*4(E, offset)
+
+ xor r0, r0 // Return success.
+
+9:
+ // Pop stack and restore registers.
+ movaps 7*16(r4), %xmm7
+ movaps 6*16(r4), %xmm6
+ movaps 5*16(r4), %xmm5
+ movaps 4*16(r4), %xmm4
+ movaps 3*16(r4), %xmm3
+ movaps 2*16(r4), %xmm2
+ movaps 1*16(r4), %xmm1
+ movaps 0*16(r4), %xmm0
+ #if 0 < LocalsSize
+ add $Padding + LocalsSize, r4
+ #endif
+ #if defined __i386__
+ pop r7
+ pop r6
+ pop r5
+ #endif
+ pop r3
+
+ ret
+
+
+ .globl _aes_decrypt_key192
+// .private_extern _aes_decrypt_key192
+_aes_decrypt_key192:
+
+ /* Save registers and set SaveSize to the number of bytes pushed onto the
+ stack so far, including the caller's return address.
+ */
+ push r3
+ #if defined __i386__
+ push r5
+ push r6
+ push r7
+ #define SaveSize (5*4)
+ #else
+ #define SaveSize (2*8)
+ #endif
+
+ /* Number of bytes used for local variables:
+
+ 8 16-byte spaces to save XMM registers.
+
+ 8 four-byte spaces for work.
+ */
+ #define LocalsSize (8*16 + 8*4)
+
+ // Define stack offset to storage space for local data.
+ #define Local (8*16)
+
+ #if 0 < LocalsSize
+ // Padding to position stack pointer at a multiple of 16 bytes.
+ #define Padding (15 & -(SaveSize + LocalsSize))
+ sub $Padding + LocalsSize, r4 // Allocate space on stack.
+ #else
+ #define Padding 0
+ #endif
+
+ /* StackFrame is the number of bytes in our stack frame, from caller's
+ stack pointer to ours (so it includes the return address).
+ */
+ #define StackFrame (SaveSize + Padding + LocalsSize)
+
+ // Save xmm registers.
+ movaps %xmm0, 0*16(r4)
+ movaps %xmm1, 1*16(r4)
+ movaps %xmm2, 2*16(r4)
+ movaps %xmm3, 3*16(r4)
+ movaps %xmm4, 4*16(r4)
+ movaps %xmm5, 5*16(r4)
+ movaps %xmm6, 6*16(r4)
+ movaps %xmm7, 7*16(r4)
+
+#if defined __i386__
+
+ // Load arguments.
+ #define Argument(i) StackFrame+4*(i)(r4)
+ mov Argument(1), E
+ mov Argument(0), K
+
+#endif
+
+// Merge point for _aes_decrypt_key and _aes_decrypt_key192.
+DKeyHas6Words:
+
+ // First words of expanded key are copied from user key.
+ movd 0*4(K), ve0
+ movd 1*4(K), ve1
+ movd 2*4(K), ve2
+ movd 3*4(K), ve3
+
+ movl $12*16, ContextKeyLength(E) // Set "key length."
+
+ #if 0 != ContextKey
+ add $ContextKey, E
+ #endif
+
+ movd 4*4(K), ve4
+ movd 5*4(K), ve5
+
+ // K cannot be used after we write to R, since they use the same register.
+
+ #if defined __i386__
+
+ lea _AESRcon, R
+ lea _AESInvMixColumnTable, ITable
+ lea _AESSubBytesWordTable, STable
+
+ #elif defined __x86_64__
+
+ lea _AESRcon(%rip), R
+ lea _AESInvMixColumnTable(%rip), ITable
+ lea _AESSubBytesWordTable(%rip), STable
+
+ #endif
+
+ /* With a six-word key, there are twelve rounds (thirteen 16-byte key
+ blocks), eleven of which have InvMixColumn applied. The key expansion
+ proceeds in iterations of six four-byte words, so the termination
+ condition is a bit complicated. We set offset to the negative of 10
+ four four-byte words, and the loop branch does another iteration if
+ offset is less than or equal to zero, meaning the number of iterations
+ performed so far is less than or equal to 10. Thus, after ten
+ iterations, it branches again. After the eleventh iteration, it
+ stops. Code after the end of the loop computes the twelfth key block,
+ which does not have InvMixColumn applied.
+ */
+ mov $-10*4*4, offset
+ sub offset, E
+
+ // Store initial words of expanded key, which are copies of user's key.
+ movd ve0, 0*4(E, offset)
+ movd ve1, 1*4(E, offset)
+ movd ve2, 2*4(E, offset)
+ movd ve3, 3*4(E, offset)
+
+ /* The first four words are stored untransformed. After that, words in
+ the expanded key are transformed by InvMixColumn.
+ */
+ movd ve4, dr
+ call InvMixColumn
+ movd vt0, 4*4(E, offset)
+
+ movd ve5, dr
+ call InvMixColumn
+ movd vt0, 5*4(E, offset)
+
+/* Here is the first iteration of the key expansion. It is separate from the
+ main loop below because we need to apply InvMixColumn to each of the
+ outputs, in ve0 through ve5. In the main loop, the technique described at
+ the top of this file is used to compute the proper outputs while using
+ InvMixColumn only once.
+*/
+ add $1, R // Advance pointer.
+ movd ve5, dr // Put previous word into work register.
+ movzx (R), t0d // Get round constant.
+ movd t0d, vt0
+
+ SubWordRotWord
+ pxor vt0, ve0
+
+ // Chain to successive words.
+ pxor ve0, ve1
+ pxor ve1, ve2
+ pxor ve2, ve3
+ pxor ve3, ve4
+ pxor ve4, ve5
+
+ add $6*4, offset
+
+ /* Apply InvMixColumn to each word. The transformed values are stored in
+ the expanded key. The original values are retained in registers for
+ further computation.
+ */
+ movd ve0, dr
+ call InvMixColumn
+ movd vt0, 0*4(E, offset)
+
+ movd ve1, dr
+ call InvMixColumn
+ movd vt0, 1*4(E, offset)
+
+ movd ve2, dr
+ call InvMixColumn
+ movd vt0, 2*4(E, offset)
+
+ movd ve3, dr
+ call InvMixColumn
+ movd vt0, 3*4(E, offset)
+
+ movd (4-6)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 4*4(E, offset)
+
+ movd (5-6)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 5*4(E, offset)
+
+// Here is the main loop.
+1:
+ add $1, R // Advance pointer.
+ movd ve5, dr // Put previous word into work register.
+ movzx (R), t0d // Get round constant.
+ movd t0d, vt0
+
+ SubWordRotWord
+ pxor vt0, ve0
+
+ // Chain to successive words.
+ pxor ve0, ve1
+ pxor ve1, ve2
+ pxor ve2, ve3
+ pxor ve3, ve4
+ pxor ve4, ve5
+
+ // Apply InvMixColumn to the difference.
+ movd vt0, dr
+ call InvMixColumn
+
+ add $6*4, offset
+
+ // Chain the transformed difference to previously transformed outputs.
+ movd (0-6)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 0*4(E, offset)
+
+ movd (1-6)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 1*4(E, offset)
+
+ movd (2-6)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 2*4(E, offset)
+
+ movd (3-6)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 3*4(E, offset)
+
+ movd (4-6)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 4*4(E, offset)
+
+ movd (5-6)*4(E, offset), vt1
+ pxor vt1, vt0
+ movd vt0, 5*4(E, offset)
+
+ jle 1b
+
+// Here is the final iteration, which does not perform InvMixColumn.
+
+ movd ve5, dr // Put previous word into work register.
+ movzx 1(R), t0d // Get round constant.
+ movd t0d, vt0
+
+ SubWordRotWord
+ pxor vt0, ve0
+
+ // Chain to successive words.
+ movd ve0, 6*4(E, offset)
+ pxor ve0, ve1
+ movd ve1, 7*4(E, offset)
+ pxor ve1, ve2
+ movd ve2, 8*4(E, offset)
+ pxor ve2, ve3
+ movd ve3, 9*4(E, offset)
+
+ xor r0, r0 // Return success.
+
+ // Pop stack and restore registers.
+ movaps 7*16(r4), %xmm7
+ movaps 6*16(r4), %xmm6
+ movaps 5*16(r4), %xmm5
+ movaps 4*16(r4), %xmm4
+ movaps 3*16(r4), %xmm3
+ movaps 2*16(r4), %xmm2
+ movaps 1*16(r4), %xmm1
+ movaps 0*16(r4), %xmm0
+ #if 0 < LocalsSize
+ add $Padding + LocalsSize, r4
+ #endif
+ #if defined __i386__
+ pop r7
+ pop r6
+ pop r5
+ #endif
+ pop r3
+
+ ret
+
+
+ .globl _aes_decrypt_key256
+// .private_extern _aes_decrypt_key256
+_aes_decrypt_key256:
+
+ /* Save registers and set SaveSize to the number of bytes pushed onto the
+ stack so far, including the caller's return address.
+ */
+ push r3
+ #if defined __i386__
+ push r5
+ push r6
+ push r7
+ #define SaveSize (5*4)
+ #else
+ #define SaveSize (2*8)
+ #endif
+
+ /* Number of bytes used for local variables:
+
+ 8 16-byte spaces to save XMM registers.
+
+ 8 four-byte spaces for work.
+ */
+ #define LocalsSize (8*16 + 8*4)
+
+ // Define stack offset to storage space for local data.
+ #define Local (8*16)
+
+ #if 0 < LocalsSize
+ // Padding to position stack pointer at a multiple of 16 bytes.
+ #define Padding (15 & -(SaveSize + LocalsSize))
+ sub $Padding + LocalsSize, r4 // Allocate space on stack.
+ #else
+ #define Padding 0
+ #endif
+
+ /* StackFrame is the number of bytes in our stack frame, from caller's
+ stack pointer to ours (so it includes the return address).
+ */
+ #define StackFrame (SaveSize + Padding + LocalsSize)
+
+ // Save xmm registers.
+ movaps %xmm0, 0*16(r4)
+ movaps %xmm1, 1*16(r4)
+ movaps %xmm2, 2*16(r4)
+ movaps %xmm3, 3*16(r4)
+ movaps %xmm4, 4*16(r4)
+ movaps %xmm5, 5*16(r4)
+ movaps %xmm6, 6*16(r4)
+ movaps %xmm7, 7*16(r4)
+
+#if defined __i386__
+
+ // Load arguments.
+ #define Argument(i) StackFrame+4*(i)(r4)
+ mov Argument(1), E
+ mov Argument(0), K
+
+#endif
+
+// Merge point for _aes_decrypt_key and _aes_decrypt_key256.
+DKeyHas8Words:
+
+ // First words of expanded key are copied from user key.
+ movd 0*4(K), ve0
+ movd 1*4(K), ve1
+ movd 2*4(K), ve2
+ movd 3*4(K), ve3
+
+ movl $14*16, ContextKeyLength(E) // Set "key length."
+
+ #if 0 != ContextKey
+ add $ContextKey, E
+ #endif
+
+ // Store initial words of expanded key, which are copies of user's key.
+ movd ve0, 0*4(E)
+ movd ve1, 1*4(E)
+ movd ve2, 2*4(E)
+ movd ve3, 3*4(E)
+ movd 4*4(K), ve0
+ movd 5*4(K), ve1
+ movd 6*4(K), ve2
+ movd 7*4(K), ve3
+
+ // K cannot be used after we write to R, since they use the same register.
+
+ #if defined __i386__
+
+ lea _AESRcon, R
+ lea _AESInvMixColumnTable, ITable
+ lea _AESSubBytesWordTable, STable
+
+ #elif defined __x86_64__
+
+ lea _AESRcon(%rip), R
+ lea _AESInvMixColumnTable(%rip), ITable
+ lea _AESSubBytesWordTable(%rip), STable
+
+ #endif
+
+ /* With an eight-word key, there are fourteen rounds (fifteen 16-byte key
+ blocks), thirteen of which have InvMixColumn applied.
+ */
+ mov $-12*4*4, offset
+ sub offset, E
+
+ // Save untransformed values in stack area.
+ movd ve0, 4*4+Local(r4)
+ movd ve1, 5*4+Local(r4)
+ movd ve2, 6*4+Local(r4)
+ movd ve3, 7*4+Local(r4)
+
+ /* Apply InvMixColumn to words 4 through 7. The transformed values are
+ stored in the expanded key. The original values are saved in the stack
+ area for further computation.
+ */
+ movd ve0, dr
+ call InvMixColumn
+ movd vt0, 4*4(E, offset)
+
+ movd ve1, dr
+ call InvMixColumn
+ movd vt0, 5*4(E, offset)
+
+ movd ve2, dr
+ call InvMixColumn
+ movd vt0, 6*4(E, offset)
+
+ movd ve3, dr
+ call InvMixColumn
+ movd vt0, 7*4(E, offset)
+
+/* Here is the first iteration of the key expansion. It is separate from the
+ main loop below because we need to apply InvMixColumn to each of the
+ outputs, in ve0 through ve3. In the main loop, the technique described at
+ the top of this file is used to compute the proper outputs while using
+ InvMixColumn only once.
+*/
+ add $1, R // Advance pointer.
+ movd ve3, dr // Put previous word into work register.
+ movzx (R), t0d // Get round constant.
+ movd t0d, vt0
+
+ SubWordRotWord
+
+ add $8*4, offset
+
+ movd (0-8)*4(E, offset), ve0 // Get old word.
+ pxor vt0, ve0
+ movd ve0, 0*4+Local(r4) // Save on stack.
+ movd ve0, dr
+ call InvMixColumn
+ movd vt0, 0*4(E, offset) // Write to expanded key.
+
+ /* Chain to successive words and apply InvMixColumn to each word. The
+ transformed values are stored in the expanded key. The original
+ values are retained in local data for further computation.
+ */
+ movd (1-8)*4(E, offset), ve1 // Get old word.
+ pxor ve0, ve1 // Chain.
+ movd ve1, 1*4+Local(r4) // Save on stack.
+ movd ve1, dr
+ call InvMixColumn
+ movd vt0, 1*4(E, offset) // Write to expanded key.
+
+ movd (2-8)*4(E, offset), ve2 // Get old word.
+ pxor ve1, ve2 // Chain.
+ movd ve2, 2*4+Local(r4) // Save on stack.
+ movd ve2, dr
+ call InvMixColumn
+ movd vt0, 2*4(E, offset) // Write to expanded key.
+
+ movd (3-8)*4(E, offset), ve3 // Get old word.
+ pxor ve2, ve3 // Chain.
+ movd ve3, 3*4+Local(r4) // Save on stack.
+ movd ve3, dr
+ call InvMixColumn
+ movd vt0, 3*4(E, offset) // Write to expanded key.
+
+ movd ve3, dr // Put previous word into work register.
+ SubWord
+
+ movd 4*4+Local(r4), ve0 // Get old word.
+ pxor vt0, ve0 // Chain.
+ movd ve0, 4*4+Local(r4) // Save on stack.
+
+ movd 5*4+Local(r4), ve1 // Get old word.
+ pxor ve0, ve1 // Chain.
+ movd ve1, 5*4+Local(r4) // Save on stack.
+
+ movd 6*4+Local(r4), ve2 // Get old word.
+ pxor ve1, ve2 // Chain.
+ movd ve2, 6*4+Local(r4) // Save on stack.
+
+ movd 7*4+Local(r4), ve3 // Get old word.
+ pxor ve2, ve3 // Chain.
+ movd ve3, 7*4+Local(r4) // Save on stack.
+
+ movd vt0, dr // Move change to work register.
+ call InvMixColumn
+
+ movd (4-8)*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, 4*4(E, offset) // Write new word to expanded key.
+
+ movd (5-8)*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, 5*4(E, offset) // Write new word to expanded key.
+
+ movd (6-8)*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, 6*4(E, offset) // Write new word to expanded key.
+
+ movd (7-8)*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, 7*4(E, offset) // Write new word to expanded key.
+
+// Here is the main loop.
+1:
+ add $1, R // Advance pointer.
+ movd ve3, dr // Put previous word into work register.
+ movzx (R), t0d // Get round constant.
+ movd t0d, vt0
+
+ SubWordRotWord
+
+ movd 0*4+Local(r4), ve0 // Get old word.
+ pxor vt0, ve0
+ movd ve0, 0*4+Local(r4) // Save on stack.
+
+ // Chain to successive words.
+ movd 1*4+Local(r4), ve1 // Get old word.
+ pxor ve0, ve1 // Chain.
+ movd ve1, 1*4+Local(r4) // Save on stack.
+
+ movd 2*4+Local(r4), ve2 // Get old word.
+ pxor ve1, ve2 // Chain.
+ movd ve2, 2*4+Local(r4) // Save on stack.
+
+ movd 3*4+Local(r4), ve3 // Get old word.
+ pxor ve2, ve3 // Chain.
+ movd ve3, 3*4+Local(r4) // Save on stack.
+
+ movd vt0, dr // Move change to work register.
+ call InvMixColumn
+
+ movd 0*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, (0+8)*4(E, offset) // Write new word to expanded key.
+
+ movd 1*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, (1+8)*4(E, offset) // Write new word to expanded key.
+
+ movd 2*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, (2+8)*4(E, offset) // Write new word to expanded key.
+
+ movd 3*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, (3+8)*4(E, offset) // Write new word to expanded key.
+
+ movd ve3, dr // Put previous word into work register.
+ SubWord
+
+ movd 4*4+Local(r4), ve0 // Get old word.
+ pxor vt0, ve0 // Chain.
+ movd ve0, 4*4+Local(r4) // Save on stack.
+
+ movd 5*4+Local(r4), ve1 // Get old word.
+ pxor ve0, ve1 // Chain.
+ movd ve1, 5*4+Local(r4) // Save on stack.
+
+ movd 6*4+Local(r4), ve2 // Get old word.
+ pxor ve1, ve2 // Chain.
+ movd ve2, 6*4+Local(r4) // Save on stack.
+
+ movd 7*4+Local(r4), ve3 // Get old word.
+ pxor ve2, ve3 // Chain.
+ movd ve3, 7*4+Local(r4) // Save on stack.
+
+ movd vt0, dr // Move change to work register.
+ call InvMixColumn
+
+ movd 4*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, (4+8)*4(E, offset) // Write new word to expanded key.
+
+ movd 5*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, (5+8)*4(E, offset) // Write new word to expanded key.
+
+ movd 6*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, (6+8)*4(E, offset) // Write new word to expanded key.
+
+ movd 7*4(E, offset), vt1 // Get old word.
+ pxor vt1, vt0 // Chain.
+ movd vt0, (7+8)*4(E, offset) // Write new word to expanded key.
+
+ add $8*4, offset
+
+ jl 1b
+
+ movd ve3, dr // Put previous word into work register.
+ movzx 1(R), t0d // Get round constant.
+ movd t0d, vt0
+
+ SubWordRotWord
+
+ movd 0*4+Local(r4), ve0 // Get old word.
+ pxor vt0, ve0 // Chain.
+ movd ve0, (0+8)*4(E, offset)
+
+ // Chain to successive words.
+ movd 1*4+Local(r4), ve1 // Get old word.
+ pxor ve0, ve1 // Chain.
+ movd ve1, (1+8)*4(E, offset)
+
+ movd 2*4+Local(r4), ve2 // Get old word.
+ pxor ve1, ve2 // Chain.
+ movd ve2, (2+8)*4(E, offset)
+
+ movd 3*4+Local(r4), ve3 // Get old word.
+ pxor ve2, ve3 // Chain.
+ movd ve3, (3+8)*4(E, offset)
+
+ xor r0, r0 // Return success.
+
+ // Pop stack and restore registers.
+ movaps 7*16(r4), %xmm7
+ movaps 6*16(r4), %xmm6
+ movaps 5*16(r4), %xmm5
+ movaps 4*16(r4), %xmm4
+ movaps 3*16(r4), %xmm3
+ movaps 2*16(r4), %xmm2
+ movaps 1*16(r4), %xmm1
+ movaps 0*16(r4), %xmm0
+ #if 0 < LocalsSize
+ add $Padding + LocalsSize, r4
+ #endif
+ #if defined __i386__
+ pop r7
+ pop r6
+ pop r5
+ #endif
+ pop r3
+
+ ret
+
+
+#undef Address
+#undef Argument
+#undef E
+#undef ITable
+#undef K
+#undef Local
+#undef LocalsSize
+#undef LookupI
+#undef LookupS
+#undef Nk
+#undef Padding
+#undef R
+#undef SaveSize
+#undef STable
+#undef StackFrame
+#undef dr
+#undef drh
+#undef drl
+#undef offset
+#undef t0
+#undef t0d
+#undef ve0
+#undef ve1
+#undef ve2
+#undef ve3
+#undef ve4
+#undef ve5
+#undef vt0
+#undef vt1
projects / apple / xnu.git / blobdiff