X-Git-Url: https://git.openssl.org/?p=openssl.git;a=blobdiff_plain;f=crypto%2Fevp%2Fe_aes_cbc_hmac_sha256.c;h=c2c48f045c3131c2ba5d18d71fdafc22571a8f10;hp=e1a21b39ee2f04c1cbbc97fc0a42a3b2f27e81d9;hb=9587429fa07a34066107e926fbc8708220f058fa;hpb=b4f0abd246340b90bb3fa2646814729f0e9d049e

diff --git a/crypto/evp/e_aes_cbc_hmac_sha256.c b/crypto/evp/e_aes_cbc_hmac_sha256.c
index e1a21b39ee..c2c48f045c 100644
--- a/crypto/evp/e_aes_cbc_hmac_sha256.c
+++ b/crypto/evp/e_aes_cbc_hmac_sha256.c
@@ -201,9 +201,14 @@ static size_t tls1_1_multi_block_encrypt(EVP_AES_HMAC_SHA256 *key,
 		u32	d[32];
 		u8	c[128];	} blocks[8];
 	SHA256_MB_CTX	*ctx;
-	unsigned int	frag, last, packlen, i, x4=4*n4x;
+	unsigned int	frag, last, packlen, i, x4=4*n4x, minblocks, processed=0;
 	size_t		ret = 0;
 	u8		*IVs;
+#if defined(BSWAP8)
+	u64		seqnum;
+#endif
+
+	RAND_bytes((IVs=blocks[0].c),16*x4);	/* ask for IVs in bulk */
 
 	ctx = (SHA256_MB_CTX *)(storage+32-((size_t)storage%32));	/* align */
 
@@ -214,9 +219,26 @@ static size_t tls1_1_multi_block_encrypt(EVP_AES_HMAC_SHA256 *key,
 		last -= x4-1;
 	}
 
+	packlen = 5+16+((frag+32+16)&-16);
+
+	/* populate descriptors with pointers and IVs */
 	hash_d[0].ptr = inp;
-	for (i=1;i<x4;i++)	hash_d[i].ptr = hash_d[i-1].ptr+frag;
+	ciph_d[0].inp = inp;
+	ciph_d[0].out = out+5+16;	/* 5+16 is place for header and explicit IV */
+	memcpy(ciph_d[0].out-16,IVs,16);
+	memcpy(ciph_d[0].iv,IVs,16);	IVs += 16;
+
+	for (i=1;i<x4;i++) {
+		ciph_d[i].inp = hash_d[i].ptr = hash_d[i-1].ptr+frag;
+		ciph_d[i].out = ciph_d[i-1].out+packlen;
+		memcpy(ciph_d[i].out-16,IVs,16);
+		memcpy(ciph_d[i].iv,IVs,16);	IVs+=16;
+	}
 
+#if defined(BSWAP8)
+	memcpy(blocks[0].c,key->md.data,8);
+	seqnum = BSWAP8(blocks[0].q[0]);
+#endif
 	for (i=0;i<x4;i++) {
 		unsigned int len = (i==(x4-1)?last:frag);
 
@@ -231,7 +253,7 @@ static size_t tls1_1_multi_block_encrypt(EVP_AES_HMAC_SHA256 *key,
 
 		/* fix seqnum */
 #if defined(BSWAP8)
-		blocks[i].q[0] = BSWAP8(BSWAP8(*(u64*)key->md.data)+i);
+		blocks[i].q[0] = BSWAP8(seqnum+i);
 #else
 		blocks[i].c[7] += ((u8*)key->md.data)[7]+i;
 		if (blocks[i].c[7] < i) {
@@ -260,6 +282,39 @@ static size_t tls1_1_multi_block_encrypt(EVP_AES_HMAC_SHA256 *key,
 	/* hash 13-byte headers and first 64-13 bytes of inputs */
 	sha256_multi_block(ctx,edges,n4x);
 	/* hash bulk inputs */
+#define	MAXCHUNKSIZE	2048
+#if	MAXCHUNKSIZE%64
+#error	"MAXCHUNKSIZE is not divisible by 64"
+#elif	MAXCHUNKSIZE
+	/* goal is to minimize pressure on L1 cache by moving
+	 * in shorter steps, so that hashed data is still in
+	 * the cache by the time we encrypt it */
+	minblocks = ((frag<=last ? frag : last)-(64-13))/64;
+	if (minblocks>MAXCHUNKSIZE/64) {
+		for (i=0;i<x4;i++) {
+			edges[i].ptr     = hash_d[i].ptr;
+			edges[i].blocks  = MAXCHUNKSIZE/64;
+			ciph_d[i].blocks = MAXCHUNKSIZE/16;
+		}
+		do {
+			sha256_multi_block(ctx,edges,n4x);
+			aesni_multi_cbc_encrypt(ciph_d,&key->ks,n4x);
+
+			for (i=0;i<x4;i++) {
+				edges[i].ptr     = hash_d[i].ptr += MAXCHUNKSIZE;
+				hash_d[i].blocks -= MAXCHUNKSIZE/64;
+				edges[i].blocks  = MAXCHUNKSIZE/64;
+				ciph_d[i].inp    += MAXCHUNKSIZE;
+				ciph_d[i].out    += MAXCHUNKSIZE;
+				ciph_d[i].blocks = MAXCHUNKSIZE/16;
+				memcpy(ciph_d[i].iv,ciph_d[i].out-16,16);
+			}
+			processed += MAXCHUNKSIZE;
+			minblocks -= MAXCHUNKSIZE/64;
+		} while (minblocks>MAXCHUNKSIZE/64);
+	}
+#endif
+#undef	MAXCHUNKSIZE
 	sha256_multi_block(ctx,hash_d,n4x);
 
 	memset(blocks,0,sizeof(blocks));
@@ -268,7 +323,7 @@ static size_t tls1_1_multi_block_encrypt(EVP_AES_HMAC_SHA256 *key,
 					off = hash_d[i].blocks*64;
 		const unsigned char    *ptr = hash_d[i].ptr+off;
 
-		off = len-(64-13)-off;	/* remainder actually */
+		off = (len-processed)-(64-13)-off;	/* remainder actually */
 		memcpy(blocks[i].c,ptr,off);
 		blocks[i].c[off]=0x80;
 		len += 64+13;		/* 64 is HMAC header */
@@ -305,23 +360,14 @@ static size_t tls1_1_multi_block_encrypt(EVP_AES_HMAC_SHA256 *key,
 	/* finalize MACs */
 	sha256_multi_block(ctx,edges,n4x);
 
-	packlen = 5+16+((frag+32+16)&-16);
-
-	out += (packlen<<(1+n4x))-packlen;
-	inp += (frag<<(1+n4x))-frag;
-
-	RAND_bytes((IVs=blocks[0].c),16*x4);	/* ask for IVs in bulk */
-
-	for (i=x4-1;;i--) {
+	for (i=0;i<x4;i++) {
 		unsigned int len = (i==(x4-1)?last:frag), pad, j;
 		unsigned char *out0 = out;
 
-		out += 5+16;		/* place for header and explicit IV */
-		ciph_d[i].inp = out;
-		ciph_d[i].out = out;
+		memcpy(ciph_d[i].out,ciph_d[i].inp,len-processed);
+		ciph_d[i].inp = ciph_d[i].out;
 
-		memmove(out,inp,len);
-		out += len;
+		out += 5+16+len;
 
 		/* write MAC */
 		((u32 *)out)[0] = BSWAP4(ctx->A[i]);
@@ -340,7 +386,7 @@ static size_t tls1_1_multi_block_encrypt(EVP_AES_HMAC_SHA256 *key,
 		for (j=0;j<=pad;j++) *(out++) = pad;
 		len += pad+1;
 
-		ciph_d[i].blocks = len/16;
+		ciph_d[i].blocks = (len-processed)/16;
 		len += 16;	/* account for explicit iv */
 
 		/* arrange header */
@@ -350,17 +396,8 @@ static size_t tls1_1_multi_block_encrypt(EVP_AES_HMAC_SHA256 *key,
 		out0[3] = (u8)(len>>8);
 		out0[4] = (u8)(len);
 
-		/* explicit iv */
-		memcpy(ciph_d[i].iv, IVs, 16);
-		memcpy(&out0[5],     IVs, 16);
-
 		ret += len+5;
-
-		if (i==0) break;
-
-		out = out0-packlen;
-		inp -= frag;
-		IVs += 16;
+		inp += frag;
 	}
 
 	aesni_multi_cbc_encrypt(ciph_d,&key->ks,n4x);
@@ -400,7 +437,7 @@ static int aesni_cbc_hmac_sha256_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 			iv = AES_BLOCK_SIZE;
 
 #if defined(STITCHED_CALL)
-		if (OPENSSL_ia32cap_P[1]&(1<<(60-32)) &&
+		if (OPENSSL_ia32cap_P[1]&(1<<(60-32)) && /* AVX? */
 		    plen>(sha_off+iv) &&
 		    (blocks=(plen-(sha_off+iv))/SHA256_CBLOCK)) {
 			SHA256_Update(&key->md,in+iv,sha_off);
@@ -451,7 +488,7 @@ static int aesni_cbc_hmac_sha256_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 		aesni_cbc_encrypt(in,out,len,
 				&key->ks,ctx->iv,0);
 
-		if (plen) {	/* "TLS" mode of operation */
+		if (plen != NO_PAYLOAD_LENGTH) {	/* "TLS" mode of operation */
 			size_t inp_len, mask, j, i;
 			unsigned int res, maxpad, pad, bitlen;
 			int ret = 1;
@@ -728,6 +765,8 @@ static int aesni_cbc_hmac_sha256_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, vo
 			}
 		}
 #if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK
+	case EVP_CTRL_TLS1_1_MULTIBLOCK_MAX_BUFSIZE:
+		return (int)(5+16+((arg+32+16)&-16));
 	case EVP_CTRL_TLS1_1_MULTIBLOCK_AAD:
 		{
 		EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM *param =
@@ -744,10 +783,17 @@ static int aesni_cbc_hmac_sha256_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, vo
 			if ((param->inp[9]<<8|param->inp[10]) < TLS1_1_VERSION)
 				return -1;
 
-			if (inp_len<4096) return 0;	/* too short */
+			if (inp_len)
+				{
+				if (inp_len<4096) return 0;	/* too short */
 
-			if (inp_len>=8192 && OPENSSL_ia32cap_P[2]&(1<<5))
-				n4x=2;	/* AVX2 */
+				if (inp_len>=8192 && OPENSSL_ia32cap_P[2]&(1<<5))
+					n4x=2;	/* AVX2 */
+				}
+			else if ((n4x=param->interleave/4) && n4x<=2)
+				inp_len = param->len;
+			else
+				return -1;
 
 			key->md = key->head;
 			SHA256_Update(&key->md,param->inp,13);