[apple/xnu.git] / bsd / crypto / aes / i386 / aes_key_hw.s

/* 	This files defines _aes_encrypt_key_hw and _aes_decrypt_key_hw --- Intel Westmere HW AES-based implementation
	of _aes_encrypt_key and _aes_decrypt_key. 

	These 2 functions SHOULD BE entried ONLY after the AES HW is verified to be available. 
	They SHOULD NOT be called without AES HW detection. It might cause xnu to crash.

	The AES HW is detected 1st thing in 
		_aes_encrypt_key (ExpandKeyForEncryption.s) 
		_aes_decrypt_key (ExpandKeyForDecryption.s)
	and, if AES HW is detected, branch without link (ie, jump) to the functions here.

	The implementation here follows the examples in an Intel White Paper
	"Intel Advanced Encryption Standard (AES) Instruction Set" Rev.2 01

	Note: Rev. 03 Final 2010 01 26 is available. Looks like some code change from Rev.2 01

	cclee 3-13-10
*/

	.text	
	.align	4,0x90

	// hw_aes_encrypt_key(key, klen, hwectx);
	// klen = 16, 24, or 32, or (128/192/256)

	.globl	_aes_encrypt_key_hw
_aes_encrypt_key_hw:

#ifdef	__i386__
	push	%ebp
	mov		%esp, %ebp
	push	%ebx
	push	%edi	
	mov		8(%ebp), %eax		// pointer to key
	mov		12(%ebp), %ebx		// klen
	mov		16(%ebp), %edi		// ctx
	#define	pkey	%eax
	#define	klen	%ebx
	#define	ctx		%edi
	#define	sp		%esp
	#define	cx		%ecx
#else
	#define	pkey	%rdi
	#define	klen	%rsi
	#define	ctx		%rdx
	#define	sp		%rsp
	#define	cx		%rcx
	push	%rbp
	mov		%rsp, %rbp
#endif

#ifdef	KERNEL
	// for xmm registers save and restore
	sub		$(16*4), sp
#endif

	cmp		$32, klen
	jg		0f					// klen>32
	shl		$3, klen			// convert 16/24/32 to 128/192/256
0:

	cmp		$128, klen			// AES-128 ?
	je		L_AES_128_Encrypt_Key
	cmp		$192, klen			// AES-192 ?
	je		L_AES_192_Encrypt_Key
	cmp		$256, klen			// AES-256 ?
	je		L_AES_256_Encrypt_Key
	mov		$1, %eax			// return error for wrong klen 
L_Encrypt_Key_2_return:
#ifdef	KERNEL
	add		$(16*4), sp
#endif
#ifdef	__i386__
	pop		%edi
	pop		%ebx
#endif
	leave
	ret

L_AES_128_Encrypt_Key:
#ifdef	KERNEL
	// save xmm registers
	movaps	%xmm1, (sp)
	movaps	%xmm2, 16(sp)
	movaps	%xmm3, 32(sp)
#endif	// KERNEL

	movl	$160, 240(ctx)		// write expanded key length to ctx
	xor		cx, cx

	movups	(pkey), %xmm1
	movups	%xmm1, (ctx)
	aeskeygenassist	$1, %xmm1, %xmm2
	call	L_key_expansion_128
	aeskeygenassist	$2, %xmm1, %xmm2
	call	L_key_expansion_128
	aeskeygenassist	$4, %xmm1, %xmm2
	call	L_key_expansion_128
	aeskeygenassist	$8, %xmm1, %xmm2
	call	L_key_expansion_128
	aeskeygenassist	$0x10, %xmm1, %xmm2
	call	L_key_expansion_128
	aeskeygenassist	$0x20, %xmm1, %xmm2
	call	L_key_expansion_128
	aeskeygenassist	$0x40, %xmm1, %xmm2
	call	L_key_expansion_128
	aeskeygenassist	$0x80, %xmm1, %xmm2
	call	L_key_expansion_128
	aeskeygenassist	$0x1b, %xmm1, %xmm2
	call	L_key_expansion_128
	aeskeygenassist	$0x36, %xmm1, %xmm2
	call	L_key_expansion_128

#ifdef	KERNEL
	// restore xmm registers
	movaps	(sp), %xmm1
	movaps	16(sp), %xmm2
	movaps	32(sp), %xmm3
#endif	// KERNEL
	xor		%eax, %eax			// return 0 for success
	jmp		L_Encrypt_Key_2_return

	.align	4, 0x90
L_key_expansion_128:
	pshufd	$0xff, %xmm2, %xmm2
	movaps	%xmm1, %xmm3
	pslldq	$4, %xmm3
	pxor	%xmm3, %xmm1
	movaps	%xmm1, %xmm3
	pslldq	$4, %xmm3
	pxor	%xmm3, %xmm1
	movaps	%xmm1, %xmm3
	pslldq	$4, %xmm3
	pxor	%xmm3, %xmm1
	pxor	%xmm2, %xmm1
	add		$16, cx
	movups	%xmm1, (ctx, cx)
	ret

L_AES_192_Encrypt_Key:
#ifdef	KERNEL
	// save xmm registers
	movaps	%xmm1, (sp)
	movaps	%xmm2, 16(sp)
	movaps	%xmm3, 32(sp)
	movaps	%xmm4, 48(sp)
#endif	// KERNEL
	movl	$192, 240(ctx)		// write expanded key length to ctx

	movups	(pkey), %xmm1
	movq	16(pkey), %xmm3

	movups	%xmm1, (ctx)
	movq	%xmm3, 16(ctx)

	lea		24(ctx), cx

	aeskeygenassist	$1, %xmm3, %xmm2
	call	L_key_expansion_192
	aeskeygenassist	$2, %xmm3, %xmm2
	call	L_key_expansion_192
	aeskeygenassist	$4, %xmm3, %xmm2
	call	L_key_expansion_192
	aeskeygenassist	$8, %xmm3, %xmm2
	call	L_key_expansion_192
	aeskeygenassist	$0x10, %xmm3, %xmm2
	call	L_key_expansion_192
	aeskeygenassist	$0x20, %xmm3, %xmm2
	call	L_key_expansion_192
	aeskeygenassist	$0x40, %xmm3, %xmm2
	call	L_key_expansion_192
	aeskeygenassist	$0x80, %xmm3, %xmm2
	call	L_key_expansion_192

#ifdef	KERNEL
	// restore xmm registers
	movaps	(sp), %xmm1
	movaps	16(sp), %xmm2
	movaps	32(sp), %xmm3
	movaps	48(sp), %xmm4
#endif	// KERNEL
	xor		%eax, %eax			// return 0 for success
	jmp		L_Encrypt_Key_2_return

	.align	4, 0x90
L_key_expansion_192:
	pshufd	$0x55, %xmm2, %xmm2

	movaps	%xmm1, %xmm4
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm1
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm1
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm1
	pxor	%xmm2, %xmm1

	pshufd	$0xff, %xmm1, %xmm2

	movaps	%xmm3, %xmm4
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm3
	pxor	%xmm2, %xmm3

	movups	%xmm1, (cx)
	movq	%xmm3, 16(cx)

	add		$24, cx
	ret

L_AES_256_Encrypt_Key:
#ifdef	KERNEL
	// save xmm registers
	movaps	%xmm1, (sp)
	movaps	%xmm2, 16(sp)
	movaps	%xmm3, 32(sp)
	movaps	%xmm4, 48(sp)
#endif	// KERNEL
	movl	$224, 240(ctx)		// write expanded key length to ctx

	movups	(pkey), %xmm1
	movups	16(pkey), %xmm3
	movups	%xmm1, (ctx)
	movups	%xmm3, 16(ctx)

	lea		32(ctx), cx

	aeskeygenassist	$1, %xmm3, %xmm2
	call	L_key_expansion_256
	aeskeygenassist	$2, %xmm3, %xmm2
	call	L_key_expansion_256
	aeskeygenassist	$4, %xmm3, %xmm2
	call	L_key_expansion_256
	aeskeygenassist	$8, %xmm3, %xmm2
	call	L_key_expansion_256
	aeskeygenassist	$0x10, %xmm3, %xmm2
	call	L_key_expansion_256
	aeskeygenassist	$0x20, %xmm3, %xmm2
	call	L_key_expansion_256
	aeskeygenassist	$0x40, %xmm3, %xmm2
	call	L_key_expansion_256_final

#ifdef	KERNEL
	// restore xmm registers
	movaps	(sp), %xmm1
	movaps	16(sp), %xmm2
	movaps	32(sp), %xmm3
	movaps	48(sp), %xmm4
#endif	// KERNEL
	xor		%eax, %eax			// return 0 for success
	jmp		L_Encrypt_Key_2_return

	.align	4, 0x90
L_key_expansion_256:

	pshufd	$0xff, %xmm2, %xmm2

	movaps	%xmm1, %xmm4
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm1
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm1
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm1
	pxor	%xmm2, %xmm1

	movups	%xmm1, (cx)

	aeskeygenassist	$0, %xmm1, %xmm4

	pshufd	$0xaa, %xmm4, %xmm2

	movaps	%xmm3, %xmm4
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm3
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm3
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm3
	pxor	%xmm2, %xmm3

	movups	%xmm3, 16(cx)

	add		$32, cx
	ret

	.align	4, 0x90
L_key_expansion_256_final:

	pshufd	$0xff, %xmm2, %xmm2

	movaps	%xmm1, %xmm4
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm1
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm1
	pslldq	$4, %xmm4

	pxor	%xmm4, %xmm1
	pxor	%xmm2, %xmm1

	movups	%xmm1, (cx)
	ret 

// _aes_decrypt_key_hw is implemented as
// 	1. call _aes_encrypt_key_hw
// 	2. use aesimc to convert the expanded round keys (except the 1st and last round keys)

	.text	
	.align	4, 0x90
	.globl	_aes_decrypt_key_hw
_aes_decrypt_key_hw:

#ifdef	__i386__

	push	%ebp
	mov		%esp, %ebp
	sub		$(8+16), %esp

	// copy input arguments for calling aes_decrypt_key_hw

	mov		8(%ebp), %eax
	mov		%eax, (%esp)
	mov		12(%ebp), %eax
	mov		%eax, 4(%esp)
	mov		16(%ebp), %eax
	mov		%eax, 8(%esp)

#else

	push	%rbp
	mov		%rsp, %rbp
	sub		$16, %rsp

	// calling arguments %rdi/%rsi/%rdx will be used for encrypt_key 
	// %rdx (ctx) will return unchanged
	// %rsi (klen) will (<<3) if <= 32

#endif
	call	_aes_encrypt_key_hw
	cmp		$0, %eax
	je		L_decrypt_inv
L_decrypt_almost_done:
#ifdef	__i386__
	add		$(8+16), %esp
#else
	add		$16, %rsp
#endif
	leave
	ret

L_decrypt_inv:
#ifdef	KERNEL
	movaps	%xmm0, (sp)
#endif

#ifdef	__i386__	
	#undef	klen
	#undef	ctx	
	mov     12(%ebp), %eax      // klen
    mov     16(%ebp), %edx      // ctx
	#define	klen	%eax
	#define	ctx		%edx
	cmp		$32, klen
	jg		0f					// klen>32
	shl		$3, klen			// convert 16/24/32 to 128/192/256
0:
#endif

	mov		$9, cx				// default is AES-128
	cmp		$128, klen
	je		L_Decrypt_Key
	add		$2, cx
	cmp		$192, klen
	je		L_Decrypt_Key
	add		$2, cx 

L_Decrypt_Key:
	add		$16, ctx
	movups	(ctx), %xmm0
	aesimc	%xmm0, %xmm0
	movups	%xmm0, (ctx)
	sub		$1, cx
	jg		L_Decrypt_Key

#ifdef	KERNEL
	movaps	(sp), %xmm0
#endif
#ifdef	__i386__
	xor		%eax, %eax
#endif
	jmp		L_decrypt_almost_done
Commit	Line	Data
6d2010ae A	1	/* This files defines _aes_encrypt_key_hw and _aes_decrypt_key_hw --- Intel Westmere HW AES-based implementation
	2	of _aes_encrypt_key and _aes_decrypt_key.
	3
	4	These 2 functions SHOULD BE entried ONLY after the AES HW is verified to be available.
	5	They SHOULD NOT be called without AES HW detection. It might cause xnu to crash.
	6
	7	The AES HW is detected 1st thing in
	8	_aes_encrypt_key (ExpandKeyForEncryption.s)
	9	_aes_decrypt_key (ExpandKeyForDecryption.s)
	10	and, if AES HW is detected, branch without link (ie, jump) to the functions here.
	11
	12	The implementation here follows the examples in an Intel White Paper
	13	"Intel Advanced Encryption Standard (AES) Instruction Set" Rev.2 01
	14
	15	Note: Rev. 03 Final 2010 01 26 is available. Looks like some code change from Rev.2 01
	16
	17	cclee 3-13-10
	18	*/
	19
	20	.text
	21	.align 4,0x90
	22
	23	// hw_aes_encrypt_key(key, klen, hwectx);
	24	// klen = 16, 24, or 32, or (128/192/256)
	25
	26	.globl _aes_encrypt_key_hw
	27	_aes_encrypt_key_hw:
	28
	29	#ifdef __i386__
	30	push %ebp
	31	mov %esp, %ebp
	32	push %ebx
	33	push %edi
	34	mov 8(%ebp), %eax // pointer to key
	35	mov 12(%ebp), %ebx // klen
	36	mov 16(%ebp), %edi // ctx
	37	#define pkey %eax
	38	#define klen %ebx
	39	#define ctx %edi
	40	#define sp %esp
	41	#define cx %ecx
	42	#else
	43	#define pkey %rdi
	44	#define klen %rsi
	45	#define ctx %rdx
	46	#define sp %rsp
	47	#define cx %rcx
	48	push %rbp
	49	mov %rsp, %rbp
	50	#endif
	51
	52	#ifdef KERNEL
	53	// for xmm registers save and restore
	54	sub $(16*4), sp
	55	#endif
	56
	57	cmp $32, klen
	58	jg 0f // klen>32
	59	shl $3, klen // convert 16/24/32 to 128/192/256
	60	0:
	61
	62	cmp $128, klen // AES-128 ?
	63	je L_AES_128_Encrypt_Key
	64	cmp $192, klen // AES-192 ?
65	je L_AES_192_Encrypt_Key
66	cmp $256, klen // AES-256 ?
67	je L_AES_256_Encrypt_Key
68	mov $1, %eax // return error for wrong klen
69	L_Encrypt_Key_2_return:
70	#ifdef KERNEL
71	add $(16*4), sp
72	#endif
73	#ifdef __i386__
74	pop %edi
75	pop %ebx
76	#endif
77	leave
78	ret
79
80	L_AES_128_Encrypt_Key:
81	#ifdef KERNEL
82	// save xmm registers
83	movaps %xmm1, (sp)
84	movaps %xmm2, 16(sp)
85	movaps %xmm3, 32(sp)
86	#endif // KERNEL
87
88	movl $160, 240(ctx) // write expanded key length to ctx
89	xor cx, cx
90
91	movups (pkey), %xmm1
92	movups %xmm1, (ctx)
93	aeskeygenassist $1, %xmm1, %xmm2
94	call L_key_expansion_128
95	aeskeygenassist $2, %xmm1, %xmm2
96	call L_key_expansion_128
97	aeskeygenassist $4, %xmm1, %xmm2
98	call L_key_expansion_128
99	aeskeygenassist $8, %xmm1, %xmm2
100	call L_key_expansion_128
101	aeskeygenassist $0x10, %xmm1, %xmm2
102	call L_key_expansion_128
103	aeskeygenassist $0x20, %xmm1, %xmm2
104	call L_key_expansion_128
105	aeskeygenassist $0x40, %xmm1, %xmm2
106	call L_key_expansion_128
107	aeskeygenassist $0x80, %xmm1, %xmm2
108	call L_key_expansion_128
109	aeskeygenassist $0x1b, %xmm1, %xmm2
110	call L_key_expansion_128
111	aeskeygenassist $0x36, %xmm1, %xmm2
112	call L_key_expansion_128
113
114	#ifdef KERNEL
115	// restore xmm registers
116	movaps (sp), %xmm1
117	movaps 16(sp), %xmm2
118	movaps 32(sp), %xmm3
119	#endif // KERNEL
120	xor %eax, %eax // return 0 for success
121	jmp L_Encrypt_Key_2_return
122
123	.align 4, 0x90
124	L_key_expansion_128:
125	pshufd $0xff, %xmm2, %xmm2
126	movaps %xmm1, %xmm3
127	pslldq $4, %xmm3
128	pxor %xmm3, %xmm1
129	movaps %xmm1, %xmm3
130	pslldq $4, %xmm3
131	pxor %xmm3, %xmm1
132	movaps %xmm1, %xmm3
133	pslldq $4, %xmm3
134	pxor %xmm3, %xmm1
135	pxor %xmm2, %xmm1
136	add $16, cx
137	movups %xmm1, (ctx, cx)
138	ret
139
140	L_AES_192_Encrypt_Key:
141	#ifdef KERNEL
142	// save xmm registers
143	movaps %xmm1, (sp)
144	movaps %xmm2, 16(sp)
145	movaps %xmm3, 32(sp)
146	movaps %xmm4, 48(sp)
147	#endif // KERNEL
148	movl $192, 240(ctx) // write expanded key length to ctx
149
150	movups (pkey), %xmm1
151	movq 16(pkey), %xmm3
152
153	movups %xmm1, (ctx)
154	movq %xmm3, 16(ctx)
155
156	lea 24(ctx), cx
157
158	aeskeygenassist $1, %xmm3, %xmm2
159	call L_key_expansion_192
160	aeskeygenassist $2, %xmm3, %xmm2
161	call L_key_expansion_192
162	aeskeygenassist $4, %xmm3, %xmm2
163	call L_key_expansion_192
164	aeskeygenassist $8, %xmm3, %xmm2
165	call L_key_expansion_192
166	aeskeygenassist $0x10, %xmm3, %xmm2
167	call L_key_expansion_192
168	aeskeygenassist $0x20, %xmm3, %xmm2
169	call L_key_expansion_192
170	aeskeygenassist $0x40, %xmm3, %xmm2
171	call L_key_expansion_192
172	aeskeygenassist $0x80, %xmm3, %xmm2
173	call L_key_expansion_192
174
175	#ifdef KERNEL
176	// restore xmm registers
177	movaps (sp), %xmm1
178	movaps 16(sp), %xmm2
179	movaps 32(sp), %xmm3
180	movaps 48(sp), %xmm4
181	#endif // KERNEL
182	xor %eax, %eax // return 0 for success
183	jmp L_Encrypt_Key_2_return
184
185	.align 4, 0x90
186	L_key_expansion_192:
187	pshufd $0x55, %xmm2, %xmm2
188
189	movaps %xmm1, %xmm4
190	pslldq $4, %xmm4
191
192	pxor %xmm4, %xmm1
193	pslldq $4, %xmm4
194
195	pxor %xmm4, %xmm1
196	pslldq $4, %xmm4
197
198	pxor %xmm4, %xmm1
199	pxor %xmm2, %xmm1
200
201	pshufd $0xff, %xmm1, %xmm2
202
203	movaps %xmm3, %xmm4
204	pslldq $4, %xmm4
205
206	pxor %xmm4, %xmm3
207	pxor %xmm2, %xmm3
208
209	movups %xmm1, (cx)
210	movq %xmm3, 16(cx)
211
212	add $24, cx
213	ret
214
215	L_AES_256_Encrypt_Key:
216	#ifdef KERNEL
217	// save xmm registers
218	movaps %xmm1, (sp)
219	movaps %xmm2, 16(sp)
220	movaps %xmm3, 32(sp)
221	movaps %xmm4, 48(sp)
222	#endif // KERNEL
223	movl $224, 240(ctx) // write expanded key length to ctx
224
225	movups (pkey), %xmm1
226	movups 16(pkey), %xmm3
227	movups %xmm1, (ctx)
228	movups %xmm3, 16(ctx)
229
230	lea 32(ctx), cx
231
232	aeskeygenassist $1, %xmm3, %xmm2
233	call L_key_expansion_256
234	aeskeygenassist $2, %xmm3, %xmm2
235	call L_key_expansion_256
236	aeskeygenassist $4, %xmm3, %xmm2
237	call L_key_expansion_256
238	aeskeygenassist $8, %xmm3, %xmm2
239	call L_key_expansion_256
240	aeskeygenassist $0x10, %xmm3, %xmm2
241	call L_key_expansion_256
242	aeskeygenassist $0x20, %xmm3, %xmm2
243	call L_key_expansion_256
244	aeskeygenassist $0x40, %xmm3, %xmm2
245	call L_key_expansion_256_final
246
247	#ifdef KERNEL
248	// restore xmm registers
249	movaps (sp), %xmm1
250	movaps 16(sp), %xmm2
251	movaps 32(sp), %xmm3
252	movaps 48(sp), %xmm4
253	#endif // KERNEL
254	xor %eax, %eax // return 0 for success
255	jmp L_Encrypt_Key_2_return
256
257	.align 4, 0x90
258	L_key_expansion_256:
259
260	pshufd $0xff, %xmm2, %xmm2
261
262	movaps %xmm1, %xmm4
263	pslldq $4, %xmm4
264
265	pxor %xmm4, %xmm1
266	pslldq $4, %xmm4
267
268	pxor %xmm4, %xmm1
269	pslldq $4, %xmm4
270
271	pxor %xmm4, %xmm1
272	pxor %xmm2, %xmm1
273
274	movups %xmm1, (cx)
275
276	aeskeygenassist $0, %xmm1, %xmm4
277
278	pshufd $0xaa, %xmm4, %xmm2
279
280	movaps %xmm3, %xmm4
281	pslldq $4, %xmm4
282
283	pxor %xmm4, %xmm3
284	pslldq $4, %xmm4
285
286	pxor %xmm4, %xmm3
287	pslldq $4, %xmm4
288
289	pxor %xmm4, %xmm3
290	pxor %xmm2, %xmm3
291
292	movups %xmm3, 16(cx)
293
294	add $32, cx
295	ret
296
297	.align 4, 0x90
298	L_key_expansion_256_final:
299
300	pshufd $0xff, %xmm2, %xmm2
301
302	movaps %xmm1, %xmm4
303	pslldq $4, %xmm4
304
305	pxor %xmm4, %xmm1
306	pslldq $4, %xmm4
307
308	pxor %xmm4, %xmm1
309	pslldq $4, %xmm4
310
311	pxor %xmm4, %xmm1
312	pxor %xmm2, %xmm1
313
314	movups %xmm1, (cx)
315	ret
316
317	// _aes_decrypt_key_hw is implemented as
318	// 1. call _aes_encrypt_key_hw
319	// 2. use aesimc to convert the expanded round keys (except the 1st and last round keys)
320
321	.text
322	.align 4, 0x90
323	.globl _aes_decrypt_key_hw
324	_aes_decrypt_key_hw:
325
326	#ifdef __i386__
327
328	push %ebp
329	mov %esp, %ebp
330	sub $(8+16), %esp
331
332	// copy input arguments for calling aes_decrypt_key_hw
333
334	mov 8(%ebp), %eax
335	mov %eax, (%esp)
336	mov 12(%ebp), %eax
337	mov %eax, 4(%esp)
338	mov 16(%ebp), %eax
339	mov %eax, 8(%esp)
340
341	#else
342
343	push %rbp
344	mov %rsp, %rbp
345	sub $16, %rsp
346
347	// calling arguments %rdi/%rsi/%rdx will be used for encrypt_key
348	// %rdx (ctx) will return unchanged
349	// %rsi (klen) will (<<3) if <= 32
350
351	#endif
352	call _aes_encrypt_key_hw
353	cmp $0, %eax
354	je L_decrypt_inv
355	L_decrypt_almost_done:
356	#ifdef __i386__
357	add $(8+16), %esp
358	#else
359	add $16, %rsp
360	#endif
361	leave
362	ret
363
364	L_decrypt_inv:
365	#ifdef KERNEL
366	movaps %xmm0, (sp)
367	#endif
368
369	#ifdef __i386__
370	#undef klen
371	#undef ctx
372	mov 12(%ebp), %eax // klen
373	mov 16(%ebp), %edx // ctx
374	#define klen %eax
375	#define ctx %edx
376	cmp $32, klen
377	jg 0f // klen>32
378	shl $3, klen // convert 16/24/32 to 128/192/256
379	0:
380	#endif
381
382	mov $9, cx // default is AES-128
383	cmp $128, klen
384	je L_Decrypt_Key
385	add $2, cx
386	cmp $192, klen
387	je L_Decrypt_Key
388	add $2, cx
389
390	L_Decrypt_Key:
391	add $16, ctx
392	movups (ctx), %xmm0
393	aesimc %xmm0, %xmm0
394	movups %xmm0, (ctx)
395	sub $1, cx
396	jg L_Decrypt_Key
397
398	#ifdef KERNEL
399	movaps (sp), %xmm0
400	#endif
401	#ifdef __i386__
402	xor %eax, %eax
403	#endif
404	jmp L_decrypt_almost_done
405