[apple/xnu.git] / libkern / crypto / sha1.c

/*
 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

/*
 * This SHA1 code is based on the basic framework from the reference
 * implementation for MD5.  That implementation is Copyright (C)
 * 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved.
 *
 * License to copy and use this software is granted provided that it
 * is identified as the "RSA Data Security, Inc. MD5 Message-Digest
 * Algorithm" in all material mentioning or referencing this software
 * or this function.
 *
 * License is also granted to make and use derivative works provided
 * that such works are identified as "derived from the RSA Data
 * Security, Inc. MD5 Message-Digest Algorithm" in all material
 * mentioning or referencing the derived work.
 *
 * RSA Data Security, Inc. makes no representations concerning either
 * the merchantability of this software or the suitability of this
 * software for any particular purpose. It is provided "as is"
 * without express or implied warranty of any kind.
 *
 * These notices must be retained in any copies of any part of this
 * documentation and/or software.
 *
 * Based on the FIPS 180-1: Secure Hash Algorithm (SHA-1) available at
 * http://www.itl.nist.gov/div897/pubs/fip180-1.htm
 */

#include <sys/types.h>
#include <sys/systm.h>
#include <libkern/OSAtomic.h>
#include <libkern/crypto/sha1.h>
#define	SHA1_TIMER	0		// change to nonzero to write timing stamps to profile sha1transform

#if SHA1_TIMER
#include <sys/kdebug.h>
#endif

#define	memset(x, y, z)	bzero(x, z);
#define	memcpy(x, y, z)	bcopy(y, x, z)

/* Internal mappings to the legacy sha1_ctxt structure. */
#define	state	h.b32
#define	bcount	c.b32
#define	buffer	m.b8

/*
 * The digest algorithm interprets the input message as a sequence of 32-bit
 * big-endian words.  We must reverse bytes in each word on x86/64 platforms,
 * but not on big-endian ones such as PPC.  For performance, we take advantage
 * of the bswap instruction on x86/64 to perform byte-reversal.  On PPC, we
 * could do 4-byte load if the address is 4-byte aligned which should further
 * improve the performance.  But for code simplicity, we punt and do 1-byte
 * loads instead.
 */
#if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__)
#define	FETCH_32(p) ({							\
	register u_int32_t l = (u_int32_t)*((const u_int32_t *)(p));	\
	__asm__ __volatile__("bswap %0" : "=r" (l) : "0" (l));		\
	l;								\
})
#else
#define	FETCH_32(p)							\
	(((u_int32_t)*((const u_int8_t *)(p) + 3)) |			\
	(((u_int32_t)*((const u_int8_t *)(p) + 2)) << 8) |		\
	(((u_int32_t)*((const u_int8_t *)(p) + 1)) << 16) |		\
	(((u_int32_t)*((const u_int8_t *)(p))) << 24))
#endif /* __i386__ || __x86_64__ */

/*
 * Encodes input (u_int32_t) into output (unsigned char). Assumes len is
 * a multiple of 4. This is not compatible with memcpy().
 */
static void
Encode(unsigned char *output, u_int32_t *input, unsigned int len)
{
	unsigned int i, j;

	for (i = 0, j = 0; j < len; i++, j += 4) {
		output[j + 3] = input[i] & 0xff;
		output[j + 2] = (input[i] >> 8) & 0xff;
		output[j + 1] = (input[i] >> 16) & 0xff;
		output[j] = (input[i] >> 24) & 0xff;
	}
}

static unsigned char PADDING[64] = { 0x80, /* zeros */ };

/* Constants from FIPS 180-1 */
#define	K_00_19		0x5a827999UL
#define	K_20_39		0x6ed9eba1UL
#define	K_40_59		0x8f1bbcdcUL
#define	K_60_79		0xca62c1d6UL

/* F, G, H and I are basic SHA1 functions. */
#define	F(b, c, d)	((((c) ^ (d)) & (b)) ^ (d))
#define	G(b, c, d)	((b) ^ (c) ^ (d))
#define	H(b, c, d)	(((b) & (c)) | (((b) | (c)) & (d)))

/* ROTATE_LEFT rotates x left n bits. */
#define	ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n))))

/* R, R1-R4 are macros used during each transformation round. */
#define R(f, k, v, w, x, y, z, i) {				\
	(v) = ROTATE_LEFT(w, 5) + f(x, y, z) + (v) + (i) + (k);	\
	(x) = ROTATE_LEFT(x, 30);				\
}

#define	R1(v, w, x, y, z, i)	R(F, K_00_19, v, w, x, y, z, i)
#define	R2(v, w, x, y, z, i)	R(G, K_20_39, v, w, x, y, z, i)
#define	R3(v, w, x, y, z, i)	R(H, K_40_59, v, w, x, y, z, i)
#define	R4(v, w, x, y, z, i)	R(G, K_60_79, v, w, x, y, z, i)

/* WUPDATE represents Wt variable that gets updated for steps 16-79 */
#define	WUPDATE(p, q, r, s) {		\
	(p) = ((q) ^ (r) ^ (s) ^ (p));	\
	(p) = ROTATE_LEFT(p, 1);	\
}

#if (defined (__x86_64__) || defined (__i386__)) 
extern void SHA1Transform(SHA1_CTX *, const u_int8_t *, u_int32_t Nblocks);
#else
static void SHA1Transform(SHA1_CTX *, const u_int8_t *);
#endif

void _SHA1Update(SHA1_CTX *context, const void *inpp, size_t inputLen);

void SHA1Final_r(SHA1_CTX *, void *);

typedef kern_return_t (*InKernelPerformSHA1Func)(void *ref, const void *data, size_t dataLen, u_int32_t *inHash, u_int32_t options, u_int32_t *outHash, Boolean usePhysicalAddress); 
void sha1_hardware_hook(Boolean option, InKernelPerformSHA1Func func, void *ref);
static void *SHA1Ref;
InKernelPerformSHA1Func performSHA1WithinKernelOnly; 
#define SHA1_USE_HARDWARE_THRESHOLD 2048 //bytes 


/*
 * SHA1 initialization. Begins a SHA1 operation, writing a new context.
 */
void
SHA1Init(SHA1_CTX *context)
{
	context->bcount[0] = context->bcount[1] = 0;
	context->count = 0;

	/* Load magic initialization constants.  */
	context->state[0] = 0x67452301UL;
	context->state[1] = 0xefcdab89UL;
	context->state[2] = 0x98badcfeUL;
	context->state[3] = 0x10325476UL;
	context->state[4] = 0xc3d2e1f0UL;
}

/*
 * SHA1 block update operation. Continues a SHA1 message-digest
 * operation, processing another message block, and updating the
 * context.
 */
void
_SHA1Update(SHA1_CTX *context, const void *inpp, size_t inputLen)
{
	u_int32_t i, index, partLen;
	const unsigned char *input = (const unsigned char *)inpp;

	if (inputLen == 0)
		return;

	/* Compute number of bytes mod 64 */
	index = (context->bcount[1] >> 3) & 0x3F;

	/* Update number of bits */
	if ((context->bcount[1] += (inputLen << 3)) < (inputLen << 3))
		context->bcount[0]++;
	context->bcount[0] += (inputLen >> 29);

	partLen = 64 - index;

	/* Transform as many times as possible. */
	i = 0;
	if (inputLen >= partLen) {
		if (index != 0) {
			memcpy(&context->buffer[index], input, partLen);
#if (defined (__x86_64__) || defined (__i386__)) 
			SHA1Transform(context, context->buffer, 1);
#else
			SHA1Transform(context, context->buffer);
#endif
			i = partLen;
		}

#if SHA1_TIMER
		KERNEL_DEBUG_CONSTANT(0xaa800004 | DBG_FUNC_START, 0, 0, 0, 0, 0);
#endif
#if (defined (__x86_64__) || defined (__i386__)) 
			{	
				int	kk = (inputLen-i)>>6;
				if (kk>0) {
					SHA1Transform(context, &input[i], kk);
					i += (kk<<6);
				}
			}
#else
		for (; i + 63 < inputLen; i += 64)
			SHA1Transform(context, &input[i]);
#endif

		 if (inputLen == i) {
#if SHA1_TIMER
		        KERNEL_DEBUG_CONSTANT(0xaa800004 | DBG_FUNC_END, 0, 0, 0, 0, 0);
#endif
			return;
		 }

		index = 0;
	}

	/* Buffer remaining input */
	memcpy(&context->buffer[index], &input[i], inputLen - i);
}


/*
 * This function is called by the SHA1 hardware kext during its init. 
 * This will register the function to call to perform SHA1 using hardware. 
 */
void sha1_hardware_hook(Boolean option, InKernelPerformSHA1Func func, void *ref)
{
	if(option) {
		// Establish the hook. The hardware is ready.
		OSCompareAndSwapPtr((void*)NULL, (void*)ref, (void * volatile*)&SHA1Ref); 

		if(!OSCompareAndSwapPtr((void *)NULL, (void *)func, (void * volatile *)&performSHA1WithinKernelOnly)) {
			panic("sha1_hardware_hook: Called twice.. Should never happen\n");
		}
	}
	else {
		// The hardware is going away. Tear down the hook. 	
		performSHA1WithinKernelOnly = NULL;
		SHA1Ref = NULL;
	}
}

static u_int32_t SHA1UpdateWithHardware(SHA1_CTX *context, const unsigned char *data, size_t dataLen, Boolean usePhysicalAddress)
{
	u_int32_t *inHashBuffer = context->state;
	u_int32_t options = 0;
	int result;

	result = performSHA1WithinKernelOnly(SHA1Ref, data, dataLen, inHashBuffer, options, inHashBuffer, usePhysicalAddress);
	if(result != KERN_SUCCESS) {
		//The hardware failed to hash for some reason. Fall back to software. 
		return 0;
	}

	//Update the context with the total length.
        /* Update number of bits */
        if ((context->bcount[1] += (dataLen << 3)) < (dataLen << 3))
                context->bcount[0]++;
        context->bcount[0] += (dataLen >> 29);
	return dataLen;
}

/*
 * This is function is only called in from the pagefault path or from page_copy().
 * So we assume that we can safely convert the virtual address to the physical address and use it.
 * Assumptions: The passed in address(inpp) is a kernel virtual address 
 * and a physical page has been faulted in. 
 * The inputLen passed in should always be less than or equal to a  page size (4096) 
 * and inpp should be on a page boundary. 
 * "performSHA1WithinKernelOnly" is initialized only when the hardware driver exists and is ready.
 */
void SHA1UpdateUsePhysicalAddress(SHA1_CTX *context, const void *inpp, size_t inputLen)
{
	Boolean usePhysicalAddress = TRUE;
	if((inputLen == PAGE_SIZE) && performSHA1WithinKernelOnly) { // If hardware exists and is ready.
		if(SHA1UpdateWithHardware(context, (const unsigned char *)inpp, inputLen, usePhysicalAddress))
			return;
		//else for some reason the hardware failed.. 
		//fall through to software and try the hash in software. 
	}
	//Use the software implementation since the hardware is absent or 
	// has not been initialized yet or inputLen !=  PAGE_SIZE. 
	_SHA1Update(context, inpp, inputLen);
}

/*
 * A wrapper around _SHA1Update() to pick between software or hardware based SHA1. 
 *
 */
void SHA1Update(SHA1_CTX *context, const void *inpp, size_t inputLen)
{
	const unsigned char *input = (const unsigned char *)inpp;
	Boolean usePhysicalAddress = FALSE;
	u_int32_t index;
	
	if((inputLen > SHA1_USE_HARDWARE_THRESHOLD) && performSHA1WithinKernelOnly) { 
		index = (context->bcount[1] >> 3) & 0x3F;
		if(index != 0) {  //bytes left in the context. Handle them first.
			u_int32_t partLen = 64 - index;
			memcpy(&context->buffer[index], input, partLen);
			_SHA1Update(context, inpp, inputLen);
			inputLen -= partLen; 
			input += partLen; 
		}
		
		u_int32_t lenForHardware = inputLen & (~0x3F); //multiple of 64
		u_int32_t bytesHashed = 0;
		bytesHashed = SHA1UpdateWithHardware(context, input, lenForHardware, usePhysicalAddress); 	
		
		inputLen -= bytesHashed;
		input += bytesHashed;
	}

	//Fall through to the software implementation.
	_SHA1Update(context, input, inputLen);
}

/*
 * For backwards compatibility, sha1_result symbol is mapped to this
 * routine since it's equivalent to SHA1Final with reversed parameters.
 */
void
SHA1Final_r(SHA1_CTX *context, void *digest)
{
	SHA1Final(digest, context);
}

/*
 * SHA1 finalization. Ends an SHA1 message-digest operation, writing the
 * the message digest and zeroizing the context.
 */
void
SHA1Final(void *digest, SHA1_CTX *context)
{
	unsigned char bits[8];
	u_int32_t index = (context->bcount[1] >> 3) & 0x3f;

	/* Save number of bits */
	Encode(bits, context->bcount, 8);

	/* Pad out to 56 mod 64. */
	SHA1Update(context, PADDING, ((index < 56) ? 56 : 120) - index);

	/* Append length (before padding) */
	SHA1Update(context, bits, 8);

	/* Store state in digest */
	Encode(digest, context->state, 20);

	/* Zeroize sensitive information. */
	memset(context, 0, sizeof (*context));
}

/*
 * SHA1 basic transformation. Transforms state based on block.
 */
#if !(defined (__x86_64__) || defined (__i386__)) 
static void
SHA1Transform(SHA1_CTX *context, const u_int8_t block[64])
{
	/* Register (instead of array) is a win in most cases */
	register u_int32_t a, b, c, d, e;
	register u_int32_t w0, w1, w2, w3, w4, w5, w6, w7;
	register u_int32_t w8, w9, w10, w11, w12, w13, w14, w15;

	a = context->state[0];
	b = context->state[1];
	c = context->state[2];
	d = context->state[3];
	e = context->state[4];

	w15 = FETCH_32(block + 60);
	w14 = FETCH_32(block + 56);
	w13 = FETCH_32(block + 52);
	w12 = FETCH_32(block + 48);
	w11 = FETCH_32(block + 44);
	w10 = FETCH_32(block + 40);
	w9  = FETCH_32(block + 36);
	w8  = FETCH_32(block + 32);
	w7  = FETCH_32(block + 28);
	w6  = FETCH_32(block + 24);
	w5  = FETCH_32(block + 20);
	w4  = FETCH_32(block + 16);
	w3  = FETCH_32(block + 12);
	w2  = FETCH_32(block +  8);
	w1  = FETCH_32(block +  4);
	w0  = FETCH_32(block +  0);

	/* Round 1 */
					R1(e, a, b, c, d,  w0);		/*  0 */
					R1(d, e, a, b, c,  w1);		/*  1 */
					R1(c, d, e, a, b,  w2);		/*  2 */
					R1(b, c, d, e, a,  w3);		/*  3 */
					R1(a, b, c, d, e,  w4);		/*  4 */
					R1(e, a, b, c, d,  w5);		/*  5 */
					R1(d, e, a, b, c,  w6);		/*  6 */
					R1(c, d, e, a, b,  w7);		/*  7 */
					R1(b, c, d, e, a,  w8);		/*  8 */
					R1(a, b, c, d, e,  w9);		/*  9 */
					R1(e, a, b, c, d, w10);		/* 10 */
					R1(d, e, a, b, c, w11);		/* 11 */
					R1(c, d, e, a, b, w12);		/* 12 */
					R1(b, c, d, e, a, w13);		/* 13 */
					R1(a, b, c, d, e, w14);		/* 14 */
					R1(e, a, b, c, d, w15);		/* 15 */
	WUPDATE( w0, w13,  w8,  w2);	R1(d, e, a, b, c,  w0);		/* 16 */
	WUPDATE( w1, w14,  w9,  w3);	R1(c, d, e, a, b,  w1);		/* 17 */
	WUPDATE( w2, w15, w10,  w4);	R1(b, c, d, e, a,  w2);		/* 18 */
	WUPDATE( w3,  w0, w11,  w5);	R1(a, b, c, d, e,  w3);		/* 19 */

	/* Round 2 */
	WUPDATE( w4,  w1, w12,  w6);	R2(e, a, b, c, d,  w4);		/* 20 */
	WUPDATE( w5,  w2, w13,  w7);	R2(d, e, a, b, c,  w5);		/* 21 */
	WUPDATE( w6,  w3, w14,  w8);	R2(c, d, e, a, b,  w6);		/* 22 */
	WUPDATE( w7,  w4, w15,  w9);	R2(b, c, d, e, a,  w7);		/* 23 */
	WUPDATE( w8,  w5,  w0, w10);	R2(a, b, c, d, e,  w8);		/* 24 */
	WUPDATE( w9,  w6,  w1, w11);	R2(e, a, b, c, d,  w9);		/* 25 */
	WUPDATE(w10,  w7,  w2, w12);	R2(d, e, a, b, c, w10);		/* 26 */
	WUPDATE(w11,  w8,  w3, w13);	R2(c, d, e, a, b, w11);		/* 27 */
	WUPDATE(w12,  w9,  w4, w14);	R2(b, c, d, e, a, w12);		/* 28 */
	WUPDATE(w13, w10,  w5, w15);	R2(a, b, c, d, e, w13);		/* 29 */
	WUPDATE(w14, w11,  w6,  w0);	R2(e, a, b, c, d, w14);		/* 30 */
	WUPDATE(w15, w12,  w7,  w1);	R2(d, e, a, b, c, w15);		/* 31 */
	WUPDATE( w0, w13,  w8,  w2);	R2(c, d, e, a, b,  w0);		/* 32 */
	WUPDATE( w1, w14,  w9,  w3);	R2(b, c, d, e, a,  w1);		/* 33 */
	WUPDATE( w2, w15, w10,  w4);	R2(a, b, c, d, e,  w2);		/* 34 */
	WUPDATE( w3,  w0, w11,  w5);	R2(e, a, b, c, d,  w3);		/* 35 */
	WUPDATE( w4,  w1, w12,  w6);	R2(d, e, a, b, c,  w4);		/* 36 */
	WUPDATE( w5,  w2, w13,  w7);	R2(c, d, e, a, b,  w5);		/* 37 */
	WUPDATE( w6,  w3, w14,  w8);	R2(b, c, d, e, a,  w6);		/* 38 */
	WUPDATE( w7,  w4, w15,  w9);	R2(a, b, c, d, e,  w7);		/* 39 */

	/* Round 3 */
	WUPDATE( w8,  w5,  w0, w10);	R3(e, a, b, c, d,  w8);		/* 40 */
	WUPDATE( w9,  w6,  w1, w11);	R3(d, e, a, b, c,  w9);		/* 41 */
	WUPDATE(w10,  w7,  w2, w12);	R3(c, d, e, a, b, w10);		/* 42 */
	WUPDATE(w11,  w8,  w3, w13);	R3(b, c, d, e, a, w11);		/* 43 */
	WUPDATE(w12,  w9,  w4, w14);	R3(a, b, c, d, e, w12);		/* 44 */
	WUPDATE(w13, w10,  w5, w15);	R3(e, a, b, c, d, w13);		/* 45 */
	WUPDATE(w14, w11,  w6,  w0);	R3(d, e, a, b, c, w14);		/* 46 */
	WUPDATE(w15, w12,  w7,  w1);	R3(c, d, e, a, b, w15);		/* 47 */
	WUPDATE( w0, w13,  w8,  w2);	R3(b, c, d, e, a,  w0);		/* 48 */
	WUPDATE( w1, w14,  w9,  w3);	R3(a, b, c, d, e,  w1);		/* 49 */
	WUPDATE( w2, w15, w10,  w4);	R3(e, a, b, c, d,  w2);		/* 50 */
	WUPDATE( w3,  w0, w11,  w5);	R3(d, e, a, b, c,  w3);		/* 51 */
	WUPDATE( w4,  w1, w12,  w6);	R3(c, d, e, a, b,  w4);		/* 52 */
	WUPDATE( w5,  w2, w13,  w7);	R3(b, c, d, e, a,  w5);		/* 53 */
	WUPDATE( w6,  w3, w14,  w8);	R3(a, b, c, d, e,  w6);		/* 54 */
	WUPDATE( w7,  w4, w15,  w9);	R3(e, a, b, c, d,  w7);		/* 55 */
	WUPDATE( w8,  w5,  w0, w10);	R3(d, e, a, b, c,  w8);		/* 56 */
	WUPDATE( w9,  w6,  w1, w11);	R3(c, d, e, a, b,  w9);		/* 57 */
	WUPDATE(w10,  w7,  w2, w12);	R3(b, c, d, e, a, w10);		/* 58 */
	WUPDATE(w11,  w8,  w3, w13);	R3(a, b, c, d, e, w11);		/* 59 */

	WUPDATE(w12,  w9,  w4, w14);	R4(e, a, b, c, d, w12);		/* 60 */
	WUPDATE(w13, w10,  w5, w15);	R4(d, e, a, b, c, w13);		/* 61 */
	WUPDATE(w14, w11,  w6,  w0);	R4(c, d, e, a, b, w14);		/* 62 */
	WUPDATE(w15, w12,  w7,  w1);	R4(b, c, d, e, a, w15);		/* 63 */
	WUPDATE( w0, w13,  w8,  w2);	R4(a, b, c, d, e,  w0);		/* 64 */
	WUPDATE( w1, w14,  w9,  w3);	R4(e, a, b, c, d,  w1);		/* 65 */
	WUPDATE( w2, w15, w10,  w4);	R4(d, e, a, b, c,  w2);		/* 66 */
	WUPDATE( w3,  w0, w11,  w5);	R4(c, d, e, a, b,  w3);		/* 67 */
	WUPDATE( w4,  w1, w12,  w6);	R4(b, c, d, e, a,  w4);		/* 68 */
	WUPDATE( w5,  w2, w13,  w7);	R4(a, b, c, d, e,  w5);		/* 69 */
	WUPDATE( w6,  w3, w14,  w8);	R4(e, a, b, c, d,  w6);		/* 70 */
	WUPDATE( w7,  w4, w15,  w9);	R4(d, e, a, b, c,  w7);		/* 71 */
	WUPDATE( w8,  w5,  w0, w10);	R4(c, d, e, a, b,  w8);		/* 72 */
	WUPDATE( w9,  w6,  w1, w11);	R4(b, c, d, e, a,  w9);		/* 73 */
	WUPDATE(w10,  w7,  w2, w12);	R4(a, b, c, d, e, w10);		/* 74 */
	WUPDATE(w11,  w8,  w3, w13);	R4(e, a, b, c, d, w11);		/* 75 */
	WUPDATE(w12,  w9,  w4, w14);	R4(d, e, a, b, c, w12);		/* 76 */
	WUPDATE(w13, w10,  w5, w15);	R4(c, d, e, a, b, w13);		/* 77 */
	WUPDATE(w14, w11,  w6,  w0);	R4(b, c, d, e, a, w14);		/* 78 */
	WUPDATE(w15, w12,  w7,  w1);	R4(a, b, c, d, e, w15);		/* 79 */

	context->state[0] += a;
	context->state[1] += b;
	context->state[2] += c;
	context->state[3] += d;
	context->state[4] += e;

	/* Zeroize sensitive information. */
	w15 = w14 = w13 = w12 = w11 = w10 = w9 = w8 = 0;
	w7 = w6 = w5 = w4 = w3 = w2 = w1 = w0 = 0;
}
#endif
Commit	Line	Data
2d21ac55 A	1	/*
	2	* Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	/*
	30	* This SHA1 code is based on the basic framework from the reference
	31	* implementation for MD5. That implementation is Copyright (C)
	32	* 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved.
	33	*
	34	* License to copy and use this software is granted provided that it
	35	* is identified as the "RSA Data Security, Inc. MD5 Message-Digest
	36	* Algorithm" in all material mentioning or referencing this software
	37	* or this function.
	38	*
	39	* License is also granted to make and use derivative works provided
	40	* that such works are identified as "derived from the RSA Data
	41	* Security, Inc. MD5 Message-Digest Algorithm" in all material
	42	* mentioning or referencing the derived work.
	43	*
	44	* RSA Data Security, Inc. makes no representations concerning either
	45	* the merchantability of this software or the suitability of this
	46	* software for any particular purpose. It is provided "as is"
	47	* without express or implied warranty of any kind.
	48	*
	49	* These notices must be retained in any copies of any part of this
	50	* documentation and/or software.
	51	*
	52	* Based on the FIPS 180-1: Secure Hash Algorithm (SHA-1) available at
	53	* http://www.itl.nist.gov/div897/pubs/fip180-1.htm
	54	*/
	55
	56	#include <sys/types.h>
	57	#include <sys/systm.h>
593a1d5f	58	#include <libkern/OSAtomic.h>
2d21ac55	59	#include <libkern/crypto/sha1.h>
6d2010ae A	60	#define SHA1_TIMER 0 // change to nonzero to write timing stamps to profile sha1transform
	61
	62	#if SHA1_TIMER
	63	#include <sys/kdebug.h>
	64	#endif
2d21ac55 A	65
	66	#define memset(x, y, z) bzero(x, z);
	67	#define memcpy(x, y, z) bcopy(y, x, z)
	68
	69	/* Internal mappings to the legacy sha1_ctxt structure. */
	70	#define state h.b32
	71	#define bcount c.b32
	72	#define buffer m.b8
	73
	74	/*
	75	* The digest algorithm interprets the input message as a sequence of 32-bit
	76	* big-endian words. We must reverse bytes in each word on x86/64 platforms,
	77	* but not on big-endian ones such as PPC. For performance, we take advantage
	78	* of the bswap instruction on x86/64 to perform byte-reversal. On PPC, we
	79	* could do 4-byte load if the address is 4-byte aligned which should further
	80	* improve the performance. But for code simplicity, we punt and do 1-byte
	81	* loads instead.
	82	*/
	83	#if (defined(__i386__) \|\| defined(__x86_64__)) && defined(__GNUC__)
	84	#define FETCH_32(p) ({ \
	85	register u_int32_t l = (u_int32_t)((const u_int32_t )(p)); \
	86	__asm__ __volatile__("bswap %0" : "=r" (l) : "0" (l)); \
	87	l; \
	88	})
	89	#else
	90	#define FETCH_32(p) \
	91	(((u_int32_t)((const u_int8_t )(p) + 3)) \| \
	92	(((u_int32_t)((const u_int8_t )(p) + 2)) << 8) \| \
	93	(((u_int32_t)((const u_int8_t )(p) + 1)) << 16) \| \
	94	(((u_int32_t)((const u_int8_t )(p))) << 24))
	95	#endif /* __i386__ \|\| __x86_64__ */
	96
	97	/*
	98	* Encodes input (u_int32_t) into output (unsigned char). Assumes len is
	99	* a multiple of 4. This is not compatible with memcpy().
	100	*/
	101	static void
	102	Encode(unsigned char output, u_int32_t input, unsigned int len)
	103	{
	104	unsigned int i, j;
	105
	106	for (i = 0, j = 0; j < len; i++, j += 4) {
	107	output[j + 3] = input[i] & 0xff;
	108	output[j + 2] = (input[i] >> 8) & 0xff;
	109	output[j + 1] = (input[i] >> 16) & 0xff;
	110	output[j] = (input[i] >> 24) & 0xff;
	111	}
	112	}
	113
	114	static unsigned char PADDING[64] = { 0x80, /* zeros */ };
	115
	116	/* Constants from FIPS 180-1 */
	117	#define K_00_19 0x5a827999UL
	118	#define K_20_39 0x6ed9eba1UL
	119	#define K_40_59 0x8f1bbcdcUL
	120	#define K_60_79 0xca62c1d6UL
	121
	122	/* F, G, H and I are basic SHA1 functions. */
	123	#define F(b, c, d) ((((c) ^ (d)) & (b)) ^ (d))
	124	#define G(b, c, d) ((b) ^ (c) ^ (d))
	125	#define H(b, c, d) (((b) & (c)) \| (((b) \| (c)) & (d)))
	126
	127	/* ROTATE_LEFT rotates x left n bits. */
	128	#define ROTATE_LEFT(x, n) (((x) << (n)) \| ((x) >> (32 - (n))))
129
130	/* R, R1-R4 are macros used during each transformation round. */
131	#define R(f, k, v, w, x, y, z, i) { \
132	(v) = ROTATE_LEFT(w, 5) + f(x, y, z) + (v) + (i) + (k); \
133	(x) = ROTATE_LEFT(x, 30); \
134	}
135
136	#define R1(v, w, x, y, z, i) R(F, K_00_19, v, w, x, y, z, i)
137	#define R2(v, w, x, y, z, i) R(G, K_20_39, v, w, x, y, z, i)
138	#define R3(v, w, x, y, z, i) R(H, K_40_59, v, w, x, y, z, i)
139	#define R4(v, w, x, y, z, i) R(G, K_60_79, v, w, x, y, z, i)
140
141	/* WUPDATE represents Wt variable that gets updated for steps 16-79 */
142	#define WUPDATE(p, q, r, s) { \
143	(p) = ((q) ^ (r) ^ (s) ^ (p)); \
144	(p) = ROTATE_LEFT(p, 1); \
145	}
146
6d2010ae A	147	#if (defined (__x86_64__) \|\| defined (__i386__))
	148	extern void SHA1Transform(SHA1_CTX , const u_int8_t , u_int32_t Nblocks);
	149	#else
	150	static void SHA1Transform(SHA1_CTX , const u_int8_t );
	151	#endif
2d21ac55	152
593a1d5f A	153	void _SHA1Update(SHA1_CTX context, const void inpp, size_t inputLen);
593a1d5f A	154
2d21ac55 A	155	void SHA1Final_r(SHA1_CTX , void );
2d21ac55 A	156
593a1d5f A	157	typedef kern_return_t (InKernelPerformSHA1Func)(void ref, const void data, size_t dataLen, u_int32_t inHash, u_int32_t options, u_int32_t *outHash, Boolean usePhysicalAddress);
	158	void sha1_hardware_hook(Boolean option, InKernelPerformSHA1Func func, void *ref);
	159	static void *SHA1Ref;
	160	InKernelPerformSHA1Func performSHA1WithinKernelOnly;
	161	#define SHA1_USE_HARDWARE_THRESHOLD 2048 //bytes
	162
	163
2d21ac55 A	164	/*
	165	* SHA1 initialization. Begins a SHA1 operation, writing a new context.
	166	*/
	167	void
	168	SHA1Init(SHA1_CTX *context)
	169	{
	170	context->bcount[0] = context->bcount[1] = 0;
	171	context->count = 0;
	172
	173	/* Load magic initialization constants. */
	174	context->state[0] = 0x67452301UL;
	175	context->state[1] = 0xefcdab89UL;
	176	context->state[2] = 0x98badcfeUL;
	177	context->state[3] = 0x10325476UL;
	178	context->state[4] = 0xc3d2e1f0UL;
	179	}
	180
	181	/*
	182	* SHA1 block update operation. Continues a SHA1 message-digest
	183	* operation, processing another message block, and updating the
	184	* context.
	185	*/
	186	void
593a1d5f	187	_SHA1Update(SHA1_CTX context, const void inpp, size_t inputLen)
2d21ac55 A	188	{
	189	u_int32_t i, index, partLen;
	190	const unsigned char input = (const unsigned char )inpp;
	191
	192	if (inputLen == 0)
	193	return;
	194
	195	/* Compute number of bytes mod 64 */
	196	index = (context->bcount[1] >> 3) & 0x3F;
	197
	198	/* Update number of bits */
	199	if ((context->bcount[1] += (inputLen << 3)) < (inputLen << 3))
	200	context->bcount[0]++;
	201	context->bcount[0] += (inputLen >> 29);
	202
	203	partLen = 64 - index;
	204
	205	/* Transform as many times as possible. */
	206	i = 0;
	207	if (inputLen >= partLen) {
	208	if (index != 0) {
	209	memcpy(&context->buffer[index], input, partLen);
6d2010ae A	210	#if (defined (__x86_64__) \|\| defined (__i386__))
	211	SHA1Transform(context, context->buffer, 1);
	212	#else
	213	SHA1Transform(context, context->buffer);
	214	#endif
2d21ac55 A	215	i = partLen;
	216	}
	217
6d2010ae A	218	#if SHA1_TIMER
	219	KERNEL_DEBUG_CONSTANT(0xaa800004 \| DBG_FUNC_START, 0, 0, 0, 0, 0);
	220	#endif
	221	#if (defined (__x86_64__) \|\| defined (__i386__))
	222	{
	223	int kk = (inputLen-i)>>6;
	224	if (kk>0) {
	225	SHA1Transform(context, &input[i], kk);
	226	i += (kk<<6);
	227	}
	228	}
	229	#else
2d21ac55	230	for (; i + 63 < inputLen; i += 64)
6d2010ae A	231	SHA1Transform(context, &input[i]);
6d2010ae A	232	#endif
2d21ac55	233
6d2010ae A	234	if (inputLen == i) {
	235	#if SHA1_TIMER
	236	KERNEL_DEBUG_CONSTANT(0xaa800004 \| DBG_FUNC_END, 0, 0, 0, 0, 0);
	237	#endif
2d21ac55	238	return;
6d2010ae	239	}
2d21ac55 A	240
	241	index = 0;
	242	}
	243
	244	/* Buffer remaining input */
	245	memcpy(&context->buffer[index], &input[i], inputLen - i);
	246	}
	247
593a1d5f A	248
	249
	250
	251	/*
	252	* This function is called by the SHA1 hardware kext during its init.
	253	* This will register the function to call to perform SHA1 using hardware.
	254	*/
	255	void sha1_hardware_hook(Boolean option, InKernelPerformSHA1Func func, void *ref)
	256	{
	257	if(option) {
	258	// Establish the hook. The hardware is ready.
b0d623f7	259	OSCompareAndSwapPtr((void)NULL, (void)ref, (void * volatile*)&SHA1Ref);
593a1d5f	260
b0d623f7	261	if(!OSCompareAndSwapPtr((void )NULL, (void )func, (void * volatile *)&performSHA1WithinKernelOnly)) {
593a1d5f A	262	panic("sha1_hardware_hook: Called twice.. Should never happen\n");
	263	}
	264	}
	265	else {
	266	// The hardware is going away. Tear down the hook.
	267	performSHA1WithinKernelOnly = NULL;
	268	SHA1Ref = NULL;
	269	}
	270	}
	271
	272	static u_int32_t SHA1UpdateWithHardware(SHA1_CTX context, const unsigned char data, size_t dataLen, Boolean usePhysicalAddress)
	273	{
	274	u_int32_t *inHashBuffer = context->state;
	275	u_int32_t options = 0;
	276	int result;
	277
	278	result = performSHA1WithinKernelOnly(SHA1Ref, data, dataLen, inHashBuffer, options, inHashBuffer, usePhysicalAddress);
	279	if(result != KERN_SUCCESS) {
	280	//The hardware failed to hash for some reason. Fall back to software.
	281	return 0;
	282	}
	283
	284	//Update the context with the total length.
	285	/* Update number of bits */
	286	if ((context->bcount[1] += (dataLen << 3)) < (dataLen << 3))
	287	context->bcount[0]++;
	288	context->bcount[0] += (dataLen >> 29);
	289	return dataLen;
	290	}
	291
	292	/*
	293	* This is function is only called in from the pagefault path or from page_copy().
	294	* So we assume that we can safely convert the virtual address to the physical address and use it.
	295	* Assumptions: The passed in address(inpp) is a kernel virtual address
	296	* and a physical page has been faulted in.
	297	* The inputLen passed in should always be less than or equal to a page size (4096)
	298	* and inpp should be on a page boundary.
	299	* "performSHA1WithinKernelOnly" is initialized only when the hardware driver exists and is ready.
	300	*/
	301	void SHA1UpdateUsePhysicalAddress(SHA1_CTX context, const void inpp, size_t inputLen)
	302	{
	303	Boolean usePhysicalAddress = TRUE;
	304	if((inputLen == PAGE_SIZE) && performSHA1WithinKernelOnly) { // If hardware exists and is ready.
	305	if(SHA1UpdateWithHardware(context, (const unsigned char *)inpp, inputLen, usePhysicalAddress))
	306	return;
	307	//else for some reason the hardware failed..
	308	//fall through to software and try the hash in software.
	309	}
	310	//Use the software implementation since the hardware is absent or
	311	// has not been initialized yet or inputLen != PAGE_SIZE.
	312	_SHA1Update(context, inpp, inputLen);
	313	}
	314
	315	/*
	316	* A wrapper around _SHA1Update() to pick between software or hardware based SHA1.
	317	*
	318	*/
	319	void SHA1Update(SHA1_CTX context, const void inpp, size_t inputLen)
	320	{
	321	const unsigned char input = (const unsigned char )inpp;
	322	Boolean usePhysicalAddress = FALSE;
	323	u_int32_t index;
	324
	325	if((inputLen > SHA1_USE_HARDWARE_THRESHOLD) && performSHA1WithinKernelOnly) {
326	index = (context->bcount[1] >> 3) & 0x3F;
327	if(index != 0) { //bytes left in the context. Handle them first.
328	u_int32_t partLen = 64 - index;
329	memcpy(&context->buffer[index], input, partLen);
330	_SHA1Update(context, inpp, inputLen);
331	inputLen -= partLen;
332	input += partLen;
333	}
334
335	u_int32_t lenForHardware = inputLen & (~0x3F); //multiple of 64
336	u_int32_t bytesHashed = 0;
337	bytesHashed = SHA1UpdateWithHardware(context, input, lenForHardware, usePhysicalAddress);
338
339	inputLen -= bytesHashed;
340	input += bytesHashed;
341	}
342
343	//Fall through to the software implementation.
344	_SHA1Update(context, input, inputLen);
345	}
346
2d21ac55 A	347	/*
	348	* For backwards compatibility, sha1_result symbol is mapped to this
	349	* routine since it's equivalent to SHA1Final with reversed parameters.
	350	*/
	351	void
	352	SHA1Final_r(SHA1_CTX context, void digest)
	353	{
	354	SHA1Final(digest, context);
	355	}
	356
	357	/*
	358	* SHA1 finalization. Ends an SHA1 message-digest operation, writing the
	359	* the message digest and zeroizing the context.
	360	*/
	361	void
	362	SHA1Final(void digest, SHA1_CTX context)
	363	{
	364	unsigned char bits[8];
	365	u_int32_t index = (context->bcount[1] >> 3) & 0x3f;
	366
	367	/* Save number of bits */
	368	Encode(bits, context->bcount, 8);
	369
	370	/* Pad out to 56 mod 64. */
	371	SHA1Update(context, PADDING, ((index < 56) ? 56 : 120) - index);
	372
	373	/* Append length (before padding) */
	374	SHA1Update(context, bits, 8);
	375
	376	/* Store state in digest */
	377	Encode(digest, context->state, 20);
	378
	379	/* Zeroize sensitive information. */
	380	memset(context, 0, sizeof (*context));
	381	}
	382
	383	/*
	384	* SHA1 basic transformation. Transforms state based on block.
	385	*/
6d2010ae	386	#if !(defined (__x86_64__) \|\| defined (__i386__))
2d21ac55	387	static void
6d2010ae	388	SHA1Transform(SHA1_CTX *context, const u_int8_t block[64])
2d21ac55 A	389	{
2d21ac55 A	390	/* Register (instead of array) is a win in most cases */
6d2010ae	391	register u_int32_t a, b, c, d, e;
2d21ac55 A	392	register u_int32_t w0, w1, w2, w3, w4, w5, w6, w7;
	393	register u_int32_t w8, w9, w10, w11, w12, w13, w14, w15;
	394
6d2010ae A	395	a = context->state[0];
	396	b = context->state[1];
	397	c = context->state[2];
	398	d = context->state[3];
	399	e = context->state[4];
	400
2d21ac55 A	401	w15 = FETCH_32(block + 60);
	402	w14 = FETCH_32(block + 56);
	403	w13 = FETCH_32(block + 52);
	404	w12 = FETCH_32(block + 48);
	405	w11 = FETCH_32(block + 44);
	406	w10 = FETCH_32(block + 40);
	407	w9 = FETCH_32(block + 36);
	408	w8 = FETCH_32(block + 32);
	409	w7 = FETCH_32(block + 28);
	410	w6 = FETCH_32(block + 24);
	411	w5 = FETCH_32(block + 20);
	412	w4 = FETCH_32(block + 16);
	413	w3 = FETCH_32(block + 12);
	414	w2 = FETCH_32(block + 8);
	415	w1 = FETCH_32(block + 4);
	416	w0 = FETCH_32(block + 0);
	417
	418	/* Round 1 */
	419	R1(e, a, b, c, d, w0); /* 0 */
	420	R1(d, e, a, b, c, w1); /* 1 */
	421	R1(c, d, e, a, b, w2); /* 2 */
	422	R1(b, c, d, e, a, w3); /* 3 */
	423	R1(a, b, c, d, e, w4); /* 4 */
	424	R1(e, a, b, c, d, w5); /* 5 */
	425	R1(d, e, a, b, c, w6); /* 6 */
	426	R1(c, d, e, a, b, w7); /* 7 */
	427	R1(b, c, d, e, a, w8); /* 8 */
	428	R1(a, b, c, d, e, w9); /* 9 */
	429	R1(e, a, b, c, d, w10); /* 10 */
	430	R1(d, e, a, b, c, w11); /* 11 */
	431	R1(c, d, e, a, b, w12); /* 12 */
	432	R1(b, c, d, e, a, w13); /* 13 */
	433	R1(a, b, c, d, e, w14); /* 14 */
	434	R1(e, a, b, c, d, w15); /* 15 */
	435	WUPDATE( w0, w13, w8, w2); R1(d, e, a, b, c, w0); /* 16 */
	436	WUPDATE( w1, w14, w9, w3); R1(c, d, e, a, b, w1); /* 17 */
	437	WUPDATE( w2, w15, w10, w4); R1(b, c, d, e, a, w2); /* 18 */
	438	WUPDATE( w3, w0, w11, w5); R1(a, b, c, d, e, w3); /* 19 */
	439
	440	/* Round 2 */
	441	WUPDATE( w4, w1, w12, w6); R2(e, a, b, c, d, w4); /* 20 */
	442	WUPDATE( w5, w2, w13, w7); R2(d, e, a, b, c, w5); /* 21 */
	443	WUPDATE( w6, w3, w14, w8); R2(c, d, e, a, b, w6); /* 22 */
	444	WUPDATE( w7, w4, w15, w9); R2(b, c, d, e, a, w7); /* 23 */
	445	WUPDATE( w8, w5, w0, w10); R2(a, b, c, d, e, w8); /* 24 */
	446	WUPDATE( w9, w6, w1, w11); R2(e, a, b, c, d, w9); /* 25 */
	447	WUPDATE(w10, w7, w2, w12); R2(d, e, a, b, c, w10); /* 26 */
	448	WUPDATE(w11, w8, w3, w13); R2(c, d, e, a, b, w11); /* 27 */
	449	WUPDATE(w12, w9, w4, w14); R2(b, c, d, e, a, w12); /* 28 */
	450	WUPDATE(w13, w10, w5, w15); R2(a, b, c, d, e, w13); /* 29 */
	451	WUPDATE(w14, w11, w6, w0); R2(e, a, b, c, d, w14); /* 30 */
	452	WUPDATE(w15, w12, w7, w1); R2(d, e, a, b, c, w15); /* 31 */
	453	WUPDATE( w0, w13, w8, w2); R2(c, d, e, a, b, w0); /* 32 */
	454	WUPDATE( w1, w14, w9, w3); R2(b, c, d, e, a, w1); /* 33 */
	455	WUPDATE( w2, w15, w10, w4); R2(a, b, c, d, e, w2); /* 34 */
	456	WUPDATE( w3, w0, w11, w5); R2(e, a, b, c, d, w3); /* 35 */
	457	WUPDATE( w4, w1, w12, w6); R2(d, e, a, b, c, w4); /* 36 */
	458	WUPDATE( w5, w2, w13, w7); R2(c, d, e, a, b, w5); /* 37 */
	459	WUPDATE( w6, w3, w14, w8); R2(b, c, d, e, a, w6); /* 38 */
	460	WUPDATE( w7, w4, w15, w9); R2(a, b, c, d, e, w7); /* 39 */
	461
	462	/* Round 3 */
	463	WUPDATE( w8, w5, w0, w10); R3(e, a, b, c, d, w8); /* 40 */
	464	WUPDATE( w9, w6, w1, w11); R3(d, e, a, b, c, w9); /* 41 */
465	WUPDATE(w10, w7, w2, w12); R3(c, d, e, a, b, w10); /* 42 */
466	WUPDATE(w11, w8, w3, w13); R3(b, c, d, e, a, w11); /* 43 */
467	WUPDATE(w12, w9, w4, w14); R3(a, b, c, d, e, w12); /* 44 */
468	WUPDATE(w13, w10, w5, w15); R3(e, a, b, c, d, w13); /* 45 */
469	WUPDATE(w14, w11, w6, w0); R3(d, e, a, b, c, w14); /* 46 */
470	WUPDATE(w15, w12, w7, w1); R3(c, d, e, a, b, w15); /* 47 */
471	WUPDATE( w0, w13, w8, w2); R3(b, c, d, e, a, w0); /* 48 */
472	WUPDATE( w1, w14, w9, w3); R3(a, b, c, d, e, w1); /* 49 */
473	WUPDATE( w2, w15, w10, w4); R3(e, a, b, c, d, w2); /* 50 */
474	WUPDATE( w3, w0, w11, w5); R3(d, e, a, b, c, w3); /* 51 */
475	WUPDATE( w4, w1, w12, w6); R3(c, d, e, a, b, w4); /* 52 */
476	WUPDATE( w5, w2, w13, w7); R3(b, c, d, e, a, w5); /* 53 */
477	WUPDATE( w6, w3, w14, w8); R3(a, b, c, d, e, w6); /* 54 */
478	WUPDATE( w7, w4, w15, w9); R3(e, a, b, c, d, w7); /* 55 */
479	WUPDATE( w8, w5, w0, w10); R3(d, e, a, b, c, w8); /* 56 */
480	WUPDATE( w9, w6, w1, w11); R3(c, d, e, a, b, w9); /* 57 */
481	WUPDATE(w10, w7, w2, w12); R3(b, c, d, e, a, w10); /* 58 */
482	WUPDATE(w11, w8, w3, w13); R3(a, b, c, d, e, w11); /* 59 */
483
484	WUPDATE(w12, w9, w4, w14); R4(e, a, b, c, d, w12); /* 60 */
485	WUPDATE(w13, w10, w5, w15); R4(d, e, a, b, c, w13); /* 61 */
486	WUPDATE(w14, w11, w6, w0); R4(c, d, e, a, b, w14); /* 62 */
487	WUPDATE(w15, w12, w7, w1); R4(b, c, d, e, a, w15); /* 63 */
488	WUPDATE( w0, w13, w8, w2); R4(a, b, c, d, e, w0); /* 64 */
489	WUPDATE( w1, w14, w9, w3); R4(e, a, b, c, d, w1); /* 65 */
490	WUPDATE( w2, w15, w10, w4); R4(d, e, a, b, c, w2); /* 66 */
491	WUPDATE( w3, w0, w11, w5); R4(c, d, e, a, b, w3); /* 67 */
492	WUPDATE( w4, w1, w12, w6); R4(b, c, d, e, a, w4); /* 68 */
493	WUPDATE( w5, w2, w13, w7); R4(a, b, c, d, e, w5); /* 69 */
494	WUPDATE( w6, w3, w14, w8); R4(e, a, b, c, d, w6); /* 70 */
495	WUPDATE( w7, w4, w15, w9); R4(d, e, a, b, c, w7); /* 71 */
496	WUPDATE( w8, w5, w0, w10); R4(c, d, e, a, b, w8); /* 72 */
497	WUPDATE( w9, w6, w1, w11); R4(b, c, d, e, a, w9); /* 73 */
498	WUPDATE(w10, w7, w2, w12); R4(a, b, c, d, e, w10); /* 74 */
499	WUPDATE(w11, w8, w3, w13); R4(e, a, b, c, d, w11); /* 75 */
500	WUPDATE(w12, w9, w4, w14); R4(d, e, a, b, c, w12); /* 76 */
501	WUPDATE(w13, w10, w5, w15); R4(c, d, e, a, b, w13); /* 77 */
502	WUPDATE(w14, w11, w6, w0); R4(b, c, d, e, a, w14); /* 78 */
503	WUPDATE(w15, w12, w7, w1); R4(a, b, c, d, e, w15); /* 79 */
504
505	context->state[0] += a;
506	context->state[1] += b;
507	context->state[2] += c;
508	context->state[3] += d;
509	context->state[4] += e;
510
511	/* Zeroize sensitive information. */
512	w15 = w14 = w13 = w12 = w11 = w10 = w9 = w8 = 0;
513	w7 = w6 = w5 = w4 = w3 = w2 = w1 = w0 = 0;
514	}
6d2010ae	515	#endif