From: Andy Polyakov <appro@openssl.org>
Date: Mon, 17 Oct 2011 17:10:54 +0000 (+0000)
Subject: Engage bsaes-x86_64.pl, bit-sliced AES.
X-Git-Tag: OpenSSL-fips-2_0-rc1~53
X-Git-Url: https://git.openssl.org/?p=openssl.git;a=commitdiff_plain;h=993adc0531cca2df70a2b663b0fcc78e3a977154

Engage bsaes-x86_64.pl, bit-sliced AES.
---

diff --git a/Configure b/Configure
index ed8c4c0bda..b624f35407 100755
--- a/Configure
+++ b/Configure
@@ -127,7 +127,7 @@ my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o x86-gf2m.o:des-586.o crypt5
 
 my $x86_elf_asm="$x86_asm:elf";
 
-my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o::aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o:e_padlock-x86_64.o";
+my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o::aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o:e_padlock-x86_64.o";
 my $ia64_asm="ia64cpuid.o:bn-ia64.o ia64-mont.o::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o rc4_skey.o:::::ghash-ia64.o::void";
 my $sparcv9_asm="sparcv9cap.o sparccpuid.o:bn-sparcv9.o sparcv9-mont.o sparcv9a-mont.o:des_enc-sparc.o fcrypt_b.o:aes_core.o aes_cbc.o aes-sparcv9.o:::sha1-sparcv9.o sha256-sparcv9.o sha512-sparcv9.o:::::::ghash-sparcv9.o::void";
 my $sparcv8_asm=":sparcv8.o:des_enc-sparc.o fcrypt_b.o:::::::::::::void";
@@ -1555,8 +1555,9 @@ if ($aes_obj =~ /\.o$/)
 	# module implements AES_ctr32_encrypt...
 	$cflags.=" -DAES_CTR_ASM" if ($aes_obj =~ s/\s*aes_ctr\.o//);
 	$aes_obj =~ s/\s*(vpaes|aesni)\-x86\.o// if ($no_sse2);
-	$aes_obj =~ s/\s*vpaes-\w*\.o// if ($fipscanisterinternal eq "y");
+	$aes_obj =~ s/\s*(vp|bs)aes-\w*\.o// if ($fipscanisterinternal eq "y");
 	$cflags.=" -DVPAES_ASM" if ($aes_obj =~ m/vpaes/);
+	$cflags.=" -DBSAES_ASM" if ($aes_obj =~ m/bsaes/);
 	}
 else	{
 	$aes_obj=$aes_enc;
diff --git a/TABLE b/TABLE
index 47b7af934b..7a95ee52e5 100644
--- a/TABLE
+++ b/TABLE
@@ -308,7 +308,7 @@ $bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
@@ -803,7 +803,7 @@ $bn_ops       = SIXTY_FOUR_BIT RC4_CHUNK_LL DES_INT EXPORT_VAR_AS_FN
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = bn_asm.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
@@ -1463,7 +1463,7 @@ $bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
@@ -1628,7 +1628,7 @@ $bn_ops       = SIXTY_FOUR_BIT RC4_CHUNK_LL DES_INT EXPORT_VAR_AS_FN
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = bn_asm.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
@@ -2420,7 +2420,7 @@ $bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
@@ -2618,7 +2618,7 @@ $bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
@@ -2684,7 +2684,7 @@ $bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
@@ -4235,7 +4235,7 @@ $bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
@@ -4400,7 +4400,7 @@ $bn_ops       = SIXTY_FOUR_BIT RC4_CHUNK_LL DES_INT EXPORT_VAR_AS_FN
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
@@ -5390,7 +5390,7 @@ $bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
@@ -5423,7 +5423,7 @@ $bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL
 $cpuid_obj    = x86_64cpuid.o
 $bn_obj       = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o
 $des_obj      = 
-$aes_obj      = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
+$aes_obj      = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o
 $bf_obj       = 
 $md5_obj      = md5-x86_64.o
 $sha1_obj     = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o
diff --git a/crypto/aes/Makefile b/crypto/aes/Makefile
index 78d5984140..86e97f335e 100644
--- a/crypto/aes/Makefile
+++ b/crypto/aes/Makefile
@@ -59,6 +59,8 @@ aes-x86_64.s: asm/aes-x86_64.pl
 	$(PERL) asm/aes-x86_64.pl $(PERLASM_SCHEME) > $@
 vpaes-x86_64.s:	asm/vpaes-x86_64.pl
 	$(PERL) asm/vpaes-x86_64.pl $(PERLASM_SCHEME) > $@
+bsaes-x86_64.s:	asm/bsaes-x86_64.pl
+	$(PERL) asm/bsaes-x86_64.pl $(PERLASM_SCHEME) > $@
 aesni-x86_64.s: asm/aesni-x86_64.pl
 	$(PERL) asm/aesni-x86_64.pl $(PERLASM_SCHEME) > $@
 aesni-sha1-x86_64.s:	asm/aesni-sha1-x86_64.pl
diff --git a/crypto/evp/e_aes.c b/crypto/evp/e_aes.c
index 4b94b2a840..53d8316484 100644
--- a/crypto/evp/e_aes.c
+++ b/crypto/evp/e_aes.c
@@ -66,11 +66,17 @@ typedef struct
 	AES_KEY ks;
 	void (*block)(const unsigned char *in, unsigned char *out,
 			const AES_KEY *key);
+	union {
 	void (*cbc)(const unsigned char *in,
 			unsigned char *out,
 			size_t length,
 			const AES_KEY *key,
 			unsigned char *ivec, int enc);
+	void (*ctr)(const unsigned char *in,
+			unsigned char *out,
+			size_t blocks, const AES_KEY *key,
+			const unsigned char ivec[16]);
+	} stream;
 
 	} EVP_AES_KEY;
 
@@ -123,6 +129,11 @@ void vpaes_cbc_encrypt(const unsigned char *in,
 			const AES_KEY *key,
 			unsigned char *ivec, int enc);
 #endif
+#ifdef BSAES_ASM
+void bsaes_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
+			size_t len, const AES_KEY *key,
+			const unsigned char ivec[16]);
+#endif
 
 #if	defined(AES_ASM) && !defined(I386_ONLY) &&	(  \
 	((defined(__i386)	|| defined(__i386__)	|| \
@@ -136,6 +147,9 @@ extern unsigned int OPENSSL_ia32cap_P[2];
 #ifdef VPAES_ASM
 #define VPAES_CAPABLE	(OPENSSL_ia32cap_P[1]&(1<<(41-32)))
 #endif
+#ifdef BSAES_ASM
+#define BSAES_CAPABLE	VPAES_CAPABLE
+#endif
 /*
  * AES-NI section
  */
@@ -690,40 +704,63 @@ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 		   const unsigned char *iv, int enc)
 	{
-	int ret;
+	int ret, mode;
 	EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
 
-	if (((ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_ECB_MODE
-	    || (ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_CBC_MODE)
+	mode = ctx->cipher->flags & EVP_CIPH_MODE;
+	if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
 	    && !enc)
 #ifdef VPAES_CAPABLE
 	    if (VPAES_CAPABLE)
 		{
 		ret = vpaes_set_decrypt_key(key,ctx->key_len*8,&dat->ks);
-		dat->block = vpaes_decrypt;
-		dat->cbc   = vpaes_cbc_encrypt;
+		dat->block	= vpaes_decrypt;
+		dat->stream.cbc	= mode==EVP_CIPH_CBC_MODE ?
+					vpaes_cbc_encrypt:NULL;
 		}
 	    else
 #endif
 		{
 		ret = AES_set_decrypt_key(key,ctx->key_len*8,&dat->ks);
-		dat->block = AES_decrypt;
-		dat->cbc   = AES_cbc_encrypt;
+		dat->block	= AES_decrypt;
+		dat->stream.cbc	= mode==EVP_CIPH_CBC_MODE ?
+					AES_cbc_encrypt:NULL;
 		}
 	else
+#ifdef BSAES_CAPABLE
+	    if (BSAES_CAPABLE && mode==EVP_CIPH_CTR_MODE)
+		{
+		ret = AES_set_encrypt_key(key,ctx->key_len*8,&dat->ks);
+		dat->block	= AES_encrypt;
+		dat->stream.ctr	= bsaes_ctr32_encrypt_blocks;
+		}
+	    else
+#endif
 #ifdef VPAES_CAPABLE
 	    if (VPAES_CAPABLE)
 		{
 		ret = vpaes_set_encrypt_key(key,ctx->key_len*8,&dat->ks);
-		dat->block = vpaes_encrypt;
-		dat->cbc   = vpaes_cbc_encrypt;
+		dat->block	= vpaes_encrypt;
+		dat->stream.cbc	= mode==EVP_CIPH_CBC_MODE ?
+					vpaes_cbc_encrypt:NULL;
 		}
 	    else
 #endif
 		{
 		ret = AES_set_encrypt_key(key,ctx->key_len*8,&dat->ks);
-		dat->block = AES_encrypt;
-		dat->cbc   = AES_cbc_encrypt;
+		dat->block	= AES_encrypt;
+		dat->stream.cbc	= mode==EVP_CIPH_CBC_MODE ?
+					AES_cbc_encrypt:NULL;
+#ifdef AES_CTR_ASM
+		if (mode==EVP_CIPH_CTR_MODE)
+			{
+			void AES_ctr32_encrypt(const unsigned char *in,
+					unsigned char *out,
+					size_t blocks, const AES_KEY *key,
+					const unsigned char ivec[AES_BLOCK_SIZE]);
+			dat->stream.ctr = AES_ctr32_encrypt;
+			}
+#endif
 		}
 
 	if(ret < 0)
@@ -740,7 +777,14 @@ static int aes_cbc_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 {
 	EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
 
-	(*dat->cbc)(in,out,len,&dat->ks,ctx->iv,ctx->encrypt);
+	if (dat->stream.cbc)
+		(*dat->stream.cbc)(in,out,len,&dat->ks,ctx->iv,ctx->encrypt);
+	else if (ctx->encrypt)
+		CRYPTO_cbc128_encrypt(in,out,len,&dat->ks,ctx->iv,
+					(block128_f)dat->block);
+	else
+		CRYPTO_cbc128_encrypt(in,out,len,&dat->ks,ctx->iv,
+					(block128_f)dat->block);
 
 	return 1;
 }
@@ -824,17 +868,13 @@ static int aes_ctr_cipher (EVP_CIPHER_CTX *ctx, unsigned char *out,
 {
 	unsigned int num = ctx->num;
 	EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
-#ifdef AES_CTR_ASM
-	void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out,
-			size_t blocks, const AES_KEY *key,
-			const unsigned char ivec[AES_BLOCK_SIZE]);
 
-	CRYPTO_ctr128_encrypt_ctr32(in,out,len,&dat->ks,
-		ctx->iv,ctx->buf,&num,(ctr128_f)AES_ctr32_encrypt);
-#else
-	CRYPTO_ctr128_encrypt(in,out,len,&dat->ks,
-		ctx->iv,ctx->buf,&num,(block128_f)dat->block);
-#endif
+	if (dat->stream.ctr)
+		CRYPTO_ctr128_encrypt_ctr32(in,out,len,&dat->ks,
+			ctx->iv,ctx->buf,&num,(ctr128_f)dat->stream.ctr);
+	else
+		CRYPTO_ctr128_encrypt(in,out,len,&dat->ks,
+			ctx->iv,ctx->buf,&num,(block128_f)dat->block);
 	ctx->num = (size_t)num;
 	return 1;
 }