xnu-1699.22.73.tar.gz

[apple/xnu.git] / bsd / crypto / aes / i386 / aes_modes_asm.s

/*\r
 ---------------------------------------------------------------------------\r
 Copyright (c) 2003, Dr Brian Gladman, Worcester, UK.   All rights reserved.\r
\r
 LICENSE TERMS\r
\r
 The free distribution and use of this software in both source and binary\r
 form is allowed (with or without changes) provided that:\r
\r
   1. distributions of this source code include the above copyright\r
      notice, this list of conditions and the following disclaimer;\r
\r
   2. distributions in binary form include the above copyright\r
      notice, this list of conditions and the following disclaimer\r
      in the documentation and/or other associated materials;\r
\r
   3. the copyright holder's name is not used to endorse products\r
      built using this software without specific written permission.\r
\r
 ALTERNATIVELY, provided that this notice is retained in full, this product\r
 may be distributed under the terms of the GNU General Public License (GPL),\r
 in which case the provisions of the GPL apply INSTEAD OF those given above.\r
\r
 DISCLAIMER\r
\r
 This software is provided 'as is' with no explicit or implied warranties\r
 in respect of its properties, including, but not limited to, correctness\r
 and/or fitness for purpose.\r
 ---------------------------------------------------------------------------\r
 Issue 31/01/2006\r
\r
 These subroutines implement multiple block AES modes for ECB, CBC, CFB,\r
 OFB and CTR encryption,  The code provides support for the VIA Advanced \r
 Cryptography Engine (ACE).\r
\r
 NOTE: In the following subroutines, the AES contexts (ctx) must be\r
 16 byte aligned if VIA ACE is being used\r
*/\r
\r
/* modified 3/5/10 cclee */\r
/* Clean up those related to VIA ACE and hand optimize aes_cbc_encrypt and aes_cbc_decrypt */\r
/* move the xmm registers save/restore originally inside the callee functions into these 2 caller functions */\r
\r
/* add code comments/description and HW AES detection and execution branch cclee 3-13-10 */\r
\r
#ifdef KERNEL\r
#include <i386/cpu_capabilities.h> 	// to use __cpu_capabilities&kHasAES to detect Intel Westmere AES HW\r
#else\r
#include <System/i386/cpu_capabilities.h> 	// to use __cpu_capabilities&kHasAES to detect Intel Westmere AES HW\r
#endif\r
\r
#if 0\r
\r
// TODO:\r
// aes_ecb_encrypt and aes_ecb_decrypt are not present in gen/aescrypt.c\r
// would add the implementation if needed\r
// they are now compiled from aes_modes.c\r
\r
aes_rval aes_ecb_encrypt(const unsigned char *ibuf, unsigned char *obuf,\r
                    int len, const aes_encrypt_ctx ctx[1])\r
{   int nb = len >> 4;\r
\r
    if(len & (AES_BLOCK_SIZE - 1)) return 1;\r
    while(nb--) {\r
        aes_encrypt(ibuf, obuf, ctx);\r
        ibuf += AES_BLOCK_SIZE;\r
        obuf += AES_BLOCK_SIZE;\r
    }\r
    return 0;\r
}\r
\r
aes_rval aes_ecb_decrypt(const unsigned char *ibuf, unsigned char *obuf,\r
                    int len, const aes_decrypt_ctx ctx[1])\r
{   int nb = len >> 4;\r
\r
    if(len & (AES_BLOCK_SIZE - 1)) return 1;\r
    while(nb--) {\r
        aes_decrypt(ibuf, obuf, ctx);\r
        ibuf += AES_BLOCK_SIZE;\r
        obuf += AES_BLOCK_SIZE;\r
    }\r
    return 0;\r
}\r
#endif\r
\r
#if 0\r
aes_rval aes_encrypt_cbc(const unsigned char *ibuf, const unsigned char *in_iv, unsigned int num_blk,\r
					 unsigned char *obuf, const aes_encrypt_ctx ctx[1])\r
{\r
		unsigned char iv[16];\r
		int i;\r
		\r
		for (i = 0; i < 16; i++) iv[i] = *(in_iv + i);\r
\r
		while (num_blk--) {\r
			iv ^= ibuf;			// 128-bit	\r
            aes_encrypt(iv, iv, ctx);\r
            memcpy(obuf, iv, AES_BLOCK_SIZE);\r
            ibuf += AES_BLOCK_SIZE;\r
            obuf += AES_BLOCK_SIZE;\r
			\r
		}		\r
\r
		return 0;\r
}\r
#endif\r
\r
	.text\r
	.align	4,0x90\r
	.globl	_aes_encrypt_cbc\r
_aes_encrypt_cbc:\r
\r
	// detect AES HW\r
	// if AES HW detected, branch to AES-HW-specific function _aes_encrypt_cbc_hw (aes_modes_hw.s)\r
	// o.w., fall through to the original AES-SW function\r
\r
#if defined	__x86_64__\r
	movq	__cpu_capabilities@GOTPCREL(%rip), %rax			// %rax -> __cpu_capability\r
	mov		(%rax), %eax									// %eax = __cpu_capabilities\r
#else\r
#ifdef KERNEL\r
	leal	__cpu_capabilities, %eax						// %eax -> __cpu_capabilities\r
	mov		(%eax), %eax									// %eax = __cpu_capabilities\r
#else\r
	mov    _COMM_PAGE_CPU_CAPABILITIES, %eax\r
#endif\r
#endif\r
	test	$(kHasAES), %eax								// kHasAES & __cpu_capabilities\r
	jne		_aes_encrypt_cbc_hw								// if AES HW detected, branch to HW-specific code\r
\r
	// save registers and allocate stack memory for xmm registers and calling arguments (i386 only)\r
#if	defined	__i386__\r
	push	%ebp\r
	mov		%esp, %ebp\r
	push	%ebx					// to be used as ibuf\r
	push	%edi					// to be used as obuf\r
	sub		$(16+16+7*16), %esp		// 12 (calling arguments) + 4 (%esi) + 16 (iv) + 7*16 (xmm)\r
	mov		%esi, 12(%esp)			// save %esp in the unused 4-bytes, to be used as num_blk\r
\r
	#define	sp	%esp\r
#else	// __x86_64__\r
	push	%rbp\r
	mov		%rsp, %rbp\r
	push	%rbx\r
	push	%r12\r
	push	%r13\r
	push	%r14\r
	push	%r15\r
	sub		$(8+16+5*16+16), %rsp	// 8 (align) + 16 (dummy iv) + 5*16 (xmm) + 16 (for i386-x86_64 consistency)	\r
\r
	#define	sp	%rsp\r
#endif\r
\r
	// save xmm registers for kernel use\r
	// xmm6-xmm7 will be used locally\r
	// xmm0-xmm2 (x86_64) or xmm0-/xmm4 (i386) will be used inside _aes_encrypt_xmm_no_save (non-restored)\r
	// there is a hole not used for xmm, which is 48(sp). \r
	// it has been used to store iv (16-bytes) in i386 code\r
	// for consistency between i386 and x86_64, this hole is dummied in x86_64 code\r
	// also the 1st 16 bytes (sp) is dummied in x86_64 code\r
\r
#ifdef	KERNEL\r
	movaps	%xmm7, 16(sp)\r
	movaps	%xmm6, 32(sp)\r
	movaps	%xmm0, 64(sp)\r
	movaps	%xmm1, 80(sp)\r
	movaps	%xmm2, 96(sp)\r
#if defined	__i386__\r
	movaps	%xmm3, 112(sp)\r
	movaps	%xmm4, 128(sp)\r
#endif\r
#endif\r
\r
	// set up registers from calling arguments\r
\r
#if defined	__i386__\r
\r
	mov		12(%ebp), %eax			// in_iv\r
	mov		24(%ebp), %edx			// ctx\r
	movups	(%eax), %xmm7			// in_iv	\r
	lea		48(%esp), %eax			// &iv[0]\r
	mov		%eax, (%esp)			// 1st iv for aes_encrypt\r
	mov		%eax, 4(%esp)			// 2nd iv for aes_encrypt\r
	mov		%edx, 8(%esp)			// ctx for aes_encrypt\r
	mov		8(%ebp), %ebx			// ibuf\r
	mov		16(%ebp), %esi			// num_blk\r
	mov		20(%ebp), %edi			// obuf\r
\r
	#define	ibuf	%ebx\r
	#define	obuf	%edi\r
	#define num_blk	%esi	\r
\r
#else	//	__x86_64__, calling arguments order : rdi/rsi/rdx/rcx/r8\r
\r
	mov		%rdi, %rbx				// ibuf\r
	lea		48(sp), %r12			// &iv\r
	movups	(%rsi), %xmm7			// in_iv\r
	mov		%rdx, %r13				// num_blk\r
	mov		%rcx, %r14				// obuf\r
	mov		%r8, %r15				// ctx	\r
\r
	#define	ibuf	%rbx\r
	#define	iv		%r12\r
	#define	num_blk	%r13d\r
	#define	obuf	%r14	\r
	#define	ctx		%r15\r
\r
#endif\r
\r
	cmp		$1, num_blk				// num_blk vs 1\r
	jl		9f						// if num_blk < 1, branch to bypass the main loop\r
0:\r
	movups	(ibuf), %xmm6			// ibuf\r
#if defined	__i386__\r
	lea		48(sp), %eax			// &iv[0]\r
	pxor	%xmm6, %xmm7			// iv ^= ibuf\r
	movups	%xmm7, (%eax)			// save iv\r
#else\r
	pxor	%xmm6, %xmm7			// iv ^= ibuf\r
	movups	%xmm7, (iv)				// save iv\r
	mov		iv, %rdi				// 1st calling argument for aes_encrypt\r
	mov		iv, %rsi				// 2nd calling argument for aes_encrypt\r
	mov		ctx, %rdx				// 3rd calling argument for aes_encrypt\r
#endif\r
	call	_aes_encrypt_xmm_no_save	// aes_encrypt(iv, iv, ctx)\r
#if defined __i386__\r
	leal	48(%esp), %eax			// &iv[0]\r
	movups	(%eax), %xmm7			// read iv\r
#else\r
	movups	(iv), %xmm7				// read iv\r
#endif\r
	movups	%xmm7, (obuf)			// memcpy(obuf, iv, AES_BLOCK_SIZE);\r
	add		$16, ibuf				// ibuf += AES_BLOCK_SIZE; \r
	add		$16, obuf				// obuf += AES_BLOCK_SIZE;	\r
	sub		$1, num_blk				// num_blk --\r
	jg		0b						// if num_blk > 0, repeat the loop\r
9:	\r
\r
L_crypt_cbc_done:\r
\r
	// restore xmm registers due to kernel use\r
#ifdef	KERNEL\r
	movaps	16(sp), %xmm7\r
	movaps	32(sp), %xmm6\r
	movaps	64(sp), %xmm0\r
	movaps	80(sp), %xmm1\r
	movaps	96(sp), %xmm2\r
#if defined	__i386__\r
	movaps	112(sp), %xmm3\r
	movaps	128(sp), %xmm4\r
#endif\r
#endif\r
\r
	xor		%eax, %eax				// to return 0 for SUCCESS\r
\r
#if	defined	__i386__\r
	mov		12(%esp), %esi			// restore %esi\r
	add		$(16+16+7*16), %esp		// 12 (calling arguments) + 4 (%esi) + 16 (iv) + 7*16 (xmm)\r
	pop		%edi\r
	pop		%ebx\r
#else\r
	add		$(8+16+5*16+16), %rsp	// 8 (align) + 16 (dummy iv) + 5*16 (xmm) + 16 (for i386-x86_64 consistency)	\r
	pop		%r15\r
	pop		%r14\r
	pop		%r13\r
	pop		%r12\r
	pop		%rbx\r
#endif\r
	leave\r
	ret\r
\r
#if 0\r
aes_rval aes_decrypt_cbc(const unsigned char *ibuf, const unsigned char *in_iv, unsigned int num_blk,\r
					 unsigned char *obuf, const aes_decrypt_ctx cx[1])\r
{\r
		unsigned char iv[16], tmp[16];\r
		int i;\r
		\r
		for (i = 0; i < 16; i++) iv[i] = *(in_iv + i);\r
\r
		while (num_blk--) {\r
\r
            memcpy(tmp, ibuf, AES_BLOCK_SIZE);\r
            aes_decrypt(ibuf, obuf, ctx);\r
			obuf ^= iv;\r
            memcpy(iv, tmp, AES_BLOCK_SIZE);\r
            ibuf += AES_BLOCK_SIZE;\r
            obuf += AES_BLOCK_SIZE;\r
		}\r
\r
		return 0;\r
}\r
#endif\r
\r
	.text\r
	.align	4,0x90\r
	.globl	_aes_decrypt_cbc\r
_aes_decrypt_cbc:\r
\r
	// detect AES HW\r
	// if AES HW detected, branch to AES-HW-specific function _aes_decrypt_cbc_hw (aes_modes_hw.s)\r
	// o.w., fall through to the original AES-SW function\r
\r
#if defined	__x86_64__\r
	movq	__cpu_capabilities@GOTPCREL(%rip), %rax			// %rax -> __cpu_capability\r
	mov		(%rax), %eax									// %eax = __cpu_capabilities\r
#else\r
#ifdef KERNEL\r
	leal	__cpu_capabilities, %eax						// %eax -> __cpu_capabilities\r
	mov		(%eax), %eax									// %eax = __cpu_capabilities\r
#else\r
	mov    _COMM_PAGE_CPU_CAPABILITIES, %eax\r
#endif\r
#endif\r
	test	$(kHasAES), %eax								// kHasAES & __cpu_capabilities\r
	jne		_aes_decrypt_cbc_hw\r
\r
	// save registers and allocate stack memory for xmm registers and calling arguments (i386 only)\r
#if	defined	__i386__\r
	push	%ebp\r
	mov		%esp, %ebp\r
	push	%ebx					// to be used as ibuf\r
	push	%edi					// to be used as obuf\r
	sub		$(16+16+7*16), %esp		// 12 (calling arguments) + 4 (%esi) + 16 (iv) + 7*16 (xmm)\r
	mov		%esi, 12(%esp)			// save %esp in the unused 4-bytes, to be used as num_blk\r
\r
	#define	sp	%esp\r
#else	// __x86_64__\r
	push	%rbp\r
	mov		%rsp, %rbp\r
	push	%rbx\r
	push	%r12\r
	push	%r13\r
	push	%r14\r
	push	%r15\r
	sub		$(8+16+5*16+16), %rsp	// 8 (align) + 16 (dummy iv) + 5*16 (xmm) + 16 (for i386-x86_64 consistency)	\r
\r
	#define	sp	%rsp\r
#endif\r
\r
	// save xmm registers for kernel use\r
	// xmm6-xmm7 will be used locally\r
	// xmm0-xmm2 (x86_64) or xmm0-/xmm4 (i386) will be used inside _aes_encrypt_xmm_no_save (non-restored)\r
	// there is a hole not used for xmm, which is 48(sp). \r
	// it has been used to store iv (16-bytes) in i386 code\r
	// for consistency between i386 and x86_64, this hole is dummied in x86_64 code\r
	// also the 1st 16 bytes (sp) is dummied in x86_64 code\r
\r
#ifdef	KERNEL\r
	movaps	%xmm7, 16(sp)\r
	movaps	%xmm6, 32(sp)\r
	movaps	%xmm0, 64(sp)\r
	movaps	%xmm1, 80(sp)\r
	movaps	%xmm2, 96(sp)\r
#if defined	__i386__\r
	movaps	%xmm3, 112(sp)\r
	movaps	%xmm4, 128(sp)\r
#endif\r
#endif\r
\r
	// set up registers from calling arguments\r
\r
#if defined	__i386__\r
	mov		12(%ebp), %eax			// in_iv\r
	mov		24(%ebp), %edx			// ctx\r
	movups	(%eax), %xmm7			// in_iv	\r
	mov		%edx, 8(%esp)			// ctx for aes_encrypt\r
	mov		8(%ebp), %ebx			// ibuf\r
	mov		16(%ebp), %esi			// num_blk\r
	mov		20(%ebp), %edi			// obuf\r
\r
	#define	ibuf	%ebx\r
	#define	obuf	%edi\r
	#define num_blk	%esi	\r
#else	//	__x86_64__, rdi/rsi/rdx/rcx/r8\r
	mov		%rdi, %rbx				// ibuf\r
	movups	(%rsi), %xmm7			// in_iv\r
	mov		%rdx, %r13				// num_blk\r
	mov		%rcx, %r14				// obuf \r
	mov		%r8, %r15				// ctx	\r
\r
	#define	ibuf	%rbx\r
	#define	num_blk	%r13d\r
	#define	obuf	%r14	\r
	#define	ctx		%r15\r
\r
#endif\r
           // memcpy(tmp, ibuf, AES_BLOCK_SIZE);\r
           // aes_decrypt(ibuf, obuf, ctx);\r
			//	obuf ^= iv;\r
           // memcpy(iv, tmp, AES_BLOCK_SIZE);\r
           // ibuf += AES_BLOCK_SIZE;\r
           // obuf += AES_BLOCK_SIZE;\r
\r
	cmp		$1, num_blk					// num_blk vs 1\r
	jl		L_crypt_cbc_done			// if num_blk < 1, bypass the main loop, jump to finishing code\r
0:\r
	movups	(ibuf), %xmm6				// tmp\r
#if defined	__i386__\r
	mov		ibuf, (sp)					// ibuf\r
	mov		obuf, 4(sp)					// obuf\r
#else\r
	mov		ibuf, %rdi					// ibuf \r
	mov		obuf, %rsi					// obuf\r
	mov		ctx, %rdx					// ctx\r
#endif\r
	call	_aes_decrypt_xmm_no_save	// aes_decrypt(ibuf, obuf, ctx)\r
	movups	(obuf), %xmm0				// obuf\r
	pxor	%xmm7, %xmm0				// obuf ^= iv;\r
	movaps	%xmm6, %xmm7				// memcpy(iv, tmp, AES_BLOCK_SIZE);\r
	movups	%xmm0, (obuf)				// update obuf\r
	add		$16, ibuf					// ibuf += AES_BLOCK_SIZE; \r
	add		$16, obuf					// obuf += AES_BLOCK_SIZE;	\r
	sub		$1, num_blk					// num_blk --\r
	jg		0b							// if num_blk > 0, repeat the loop\r
9:	\r
\r
	// we are done here, the finishing code is identical to that in aes_encrypt_cbc, so just jump to there\r
	jmp		L_crypt_cbc_done\r
\r