const unsigned char iv[16]);
#endif
+#if defined(VPAES_ASM) && (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC))
+extern unsigned int OPENSSL_ppccap_P;
+#define VPAES_CAPABLE (OPENSSL_ppccap_P&(1<<1))
+#endif
+
#if defined(AES_ASM) && !defined(I386_ONLY) && ( \
((defined(__i386) || defined(__i386__) || \
defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \
const unsigned char ivec[16],
unsigned char cmac[16]);
+#if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
+size_t aesni_gcm_encrypt(const unsigned char *in,
+ unsigned char *out,
+ size_t len,
+ const void *key,
+ unsigned char ivec[16],
+ u64 *Xi);
+#define AES_gcm_encrypt aesni_gcm_encrypt
+size_t aesni_gcm_decrypt(const unsigned char *in,
+ unsigned char *out,
+ size_t len,
+ const void *key,
+ unsigned char ivec[16],
+ u64 *Xi);
+#define AES_gcm_decrypt aesni_gcm_decrypt
+void gcm_ghash_avx(u64 Xi[2],const u128 Htable[16],const u8 *in,size_t len);
+#define AES_GCM_ASM(gctx) (gctx->ctr==aesni_ctr32_encrypt_blocks && \
+ gctx->gcm.ghash==gcm_ghash_avx)
+#define AES_GCM_ASM2(gctx) (gctx->gcm.block==(block128_f)aesni_encrypt && \
+ gctx->gcm.ghash==gcm_ghash_avx)
+#undef AES_GCM_ASM2 /* minor size optimization */
+#endif
+
static int aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
const unsigned char *iv, int enc)
{
void aes256_t4_ctr32_encrypt (const unsigned char *in, unsigned char *out,
size_t blocks, const AES_KEY *key,
unsigned char *ivec);
+void aes128_t4_xts_encrypt (const unsigned char *in, unsigned char *out,
+ size_t blocks, const AES_KEY *key1,
+ const AES_KEY *key2, const unsigned char *ivec);
+void aes128_t4_xts_decrypt (const unsigned char *in, unsigned char *out,
+ size_t blocks, const AES_KEY *key1,
+ const AES_KEY *key2, const unsigned char *ivec);
+void aes256_t4_xts_encrypt (const unsigned char *in, unsigned char *out,
+ size_t blocks, const AES_KEY *key1,
+ const AES_KEY *key2, const unsigned char *ivec);
+void aes256_t4_xts_decrypt (const unsigned char *in, unsigned char *out,
+ size_t blocks, const AES_KEY *key1,
+ const AES_KEY *key2, const unsigned char *ivec);
static int aes_t4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
const unsigned char *iv, int enc)
if(ret < 0)
{
- EVPerr(EVP_F_AES_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED);
+ EVPerr(EVP_F_AES_T4_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED);
return 0;
}
if (key)
{
int bits = ctx->key_len * 4;
+ xctx->stream = NULL;
/* key_len is two AES keys */
if (enc)
{
aes_t4_set_encrypt_key(key, bits, &xctx->ks1.ks);
xctx->xts.block1 = (block128_f)aes_t4_encrypt;
-#if 0 /* not yet */
switch (bits) {
case 128:
xctx->stream = aes128_t4_xts_encrypt;
break;
+#if 0 /* not yet */
case 192:
xctx->stream = aes192_t4_xts_encrypt;
break;
+#endif
case 256:
xctx->stream = aes256_t4_xts_encrypt;
break;
default:
return 0;
}
-#endif
}
else
{
aes_t4_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
xctx->xts.block1 = (block128_f)aes_t4_decrypt;
-#if 0 /* not yet */
switch (bits) {
case 128:
xctx->stream = aes128_t4_xts_decrypt;
break;
+#if 0 /* not yet */
case 192:
xctx->stream = aes192_t4_xts_decrypt;
break;
+#endif
case 256:
xctx->stream = aes256_t4_xts_decrypt;
break;
default:
return 0;
}
-#endif
}
aes_t4_set_encrypt_key(key + ctx->key_len/2,
NULL,NULL,aes_##mode##_ctrl,NULL }; \
const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
{ return &aes_##keylen##_##mode; }
+
+#endif
+
+#if defined(OPENSSL_CPUID_OBJ) && (defined(__arm__) || defined(__arm) || defined(__aarch64__))
+#include "arm_arch.h"
+#if __ARM_ARCH__>=7
+# if defined(BSAES_ASM)
+# define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
+# endif
+# define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
+# define HWAES_set_encrypt_key aes_v8_set_encrypt_key
+# define HWAES_set_decrypt_key aes_v8_set_decrypt_key
+# define HWAES_encrypt aes_v8_encrypt
+# define HWAES_decrypt aes_v8_decrypt
+# define HWAES_cbc_encrypt aes_v8_cbc_encrypt
+# define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
+#endif
+#endif
+
+#if defined(HWAES_CAPABLE)
+int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits,
+ AES_KEY *key);
+int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits,
+ AES_KEY *key);
+void HWAES_encrypt(const unsigned char *in, unsigned char *out,
+ const AES_KEY *key);
+void HWAES_decrypt(const unsigned char *in, unsigned char *out,
+ const AES_KEY *key);
+void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out,
+ size_t length, const AES_KEY *key,
+ unsigned char *ivec, const int enc);
+void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
+ size_t len, const AES_KEY *key, const unsigned char ivec[16]);
#endif
#define BLOCK_CIPHER_generic_pack(nid,keylen,flags) \
mode = ctx->cipher->flags & EVP_CIPH_MODE;
if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
&& !enc)
+#ifdef HWAES_CAPABLE
+ if (HWAES_CAPABLE)
+ {
+ ret = HWAES_set_decrypt_key(key,ctx->key_len*8,&dat->ks.ks);
+ dat->block = (block128_f)HWAES_decrypt;
+ dat->stream.cbc = NULL;
+#ifdef HWAES_cbc_encrypt
+ if (mode==EVP_CIPH_CBC_MODE)
+ dat->stream.cbc = (cbc128_f)HWAES_cbc_encrypt;
+#endif
+ }
+ else
+#endif
#ifdef BSAES_CAPABLE
if (BSAES_CAPABLE && mode==EVP_CIPH_CBC_MODE)
{
NULL;
}
else
+#ifdef HWAES_CAPABLE
+ if (HWAES_CAPABLE)
+ {
+ ret = HWAES_set_encrypt_key(key,ctx->key_len*8,&dat->ks.ks);
+ dat->block = (block128_f)HWAES_encrypt;
+ dat->stream.cbc = NULL;
+#ifdef HWAES_cbc_encrypt
+ if (mode==EVP_CIPH_CBC_MODE)
+ dat->stream.cbc = (cbc128_f)HWAES_cbc_encrypt;
+ else
+#endif
+#ifdef HWAES_ctr32_encrypt_blocks
+ if (mode==EVP_CIPH_CTR_MODE)
+ dat->stream.ctr = (ctr128_f)HWAES_ctr32_encrypt_blocks;
+ else
+#endif
+ (void)0; /* terminate potentially open 'else' */
+ }
+ else
+#endif
#ifdef BSAES_CAPABLE
if (BSAES_CAPABLE && mode==EVP_CIPH_CTR_MODE)
{
else if (ctx->encrypt)
CRYPTO_cbc128_encrypt(in,out,len,&dat->ks,ctx->iv,dat->block);
else
- CRYPTO_cbc128_encrypt(in,out,len,&dat->ks,ctx->iv,dat->block);
+ CRYPTO_cbc128_decrypt(in,out,len,&dat->ks,ctx->iv,dat->block);
return 1;
}
gctx->ctr = NULL;
break;
}
+ else
#endif
+ (void)0; /* terminate potentially open 'else' */
+
AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f)AES_encrypt);
#ifdef AES_CTR_ASM
/* Encrypt payload */
if (gctx->ctr)
{
+ size_t bulk=0;
+#if defined(AES_GCM_ASM)
+ if (len>=32 && AES_GCM_ASM(gctx))
+ {
+ if (CRYPTO_gcm128_encrypt(&gctx->gcm,NULL,NULL,0))
+ return -1;
+
+ bulk = AES_gcm_encrypt(in,out,len,
+ gctx->gcm.key,
+ gctx->gcm.Yi.c,
+ gctx->gcm.Xi.u);
+ gctx->gcm.len.u[1] += bulk;
+ }
+#endif
if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
- in, out, len,
+ in +bulk,
+ out+bulk,
+ len-bulk,
gctx->ctr))
goto err;
}
else {
- if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, len))
+ size_t bulk=0;
+#if defined(AES_GCM_ASM2)
+ if (len>=32 && AES_GCM_ASM2(gctx))
+ {
+ if (CRYPTO_gcm128_encrypt(&gctx->gcm,NULL,NULL,0))
+ return -1;
+
+ bulk = AES_gcm_encrypt(in,out,len,
+ gctx->gcm.key,
+ gctx->gcm.Yi.c,
+ gctx->gcm.Xi.u);
+ gctx->gcm.len.u[1] += bulk;
+ }
+#endif
+ if (CRYPTO_gcm128_encrypt(&gctx->gcm,
+ in +bulk,
+ out+bulk,
+ len-bulk))
goto err;
}
out += len;
/* Decrypt */
if (gctx->ctr)
{
+ size_t bulk=0;
+#if defined(AES_GCM_ASM)
+ if (len>=16 && AES_GCM_ASM(gctx))
+ {
+ if (CRYPTO_gcm128_decrypt(&gctx->gcm,NULL,NULL,0))
+ return -1;
+
+ bulk = AES_gcm_decrypt(in,out,len,
+ gctx->gcm.key,
+ gctx->gcm.Yi.c,
+ gctx->gcm.Xi.u);
+ gctx->gcm.len.u[1] += bulk;
+ }
+#endif
if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
- in, out, len,
+ in +bulk,
+ out+bulk,
+ len-bulk,
gctx->ctr))
goto err;
}
else {
- if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, len))
+ size_t bulk=0;
+#if defined(AES_GCM_ASM2)
+ if (len>=16 && AES_GCM_ASM2(gctx))
+ {
+ if (CRYPTO_gcm128_decrypt(&gctx->gcm,NULL,NULL,0))
+ return -1;
+
+ bulk = AES_gcm_decrypt(in,out,len,
+ gctx->gcm.key,
+ gctx->gcm.Yi.c,
+ gctx->gcm.Xi.u);
+ gctx->gcm.len.u[1] += bulk;
+ }
+#endif
+ if (CRYPTO_gcm128_decrypt(&gctx->gcm,
+ in +bulk,
+ out+bulk,
+ len-bulk))
goto err;
}
/* Retrieve tag */
{
if (gctx->ctr)
{
+ size_t bulk=0;
+#if defined(AES_GCM_ASM)
+ if (len>=32 && AES_GCM_ASM(gctx))
+ {
+ size_t res = (16-gctx->gcm.mres)%16;
+
+ if (CRYPTO_gcm128_encrypt(&gctx->gcm,
+ in,out,res))
+ return -1;
+
+ bulk = AES_gcm_encrypt(in+res,
+ out+res,len-res, gctx->gcm.key,
+ gctx->gcm.Yi.c,
+ gctx->gcm.Xi.u);
+ gctx->gcm.len.u[1] += bulk;
+ bulk += res;
+ }
+#endif
if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
- in, out, len,
+ in +bulk,
+ out+bulk,
+ len-bulk,
gctx->ctr))
return -1;
}
else {
- if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, len))
+ size_t bulk=0;
+#if defined(AES_GCM_ASM2)
+ if (len>=32 && AES_GCM_ASM2(gctx))
+ {
+ size_t res = (16-gctx->gcm.mres)%16;
+
+ if (CRYPTO_gcm128_encrypt(&gctx->gcm,
+ in,out,res))
+ return -1;
+
+ bulk = AES_gcm_encrypt(in+res,
+ out+res,len-res, gctx->gcm.key,
+ gctx->gcm.Yi.c,
+ gctx->gcm.Xi.u);
+ gctx->gcm.len.u[1] += bulk;
+ bulk += res;
+ }
+#endif
+ if (CRYPTO_gcm128_encrypt(&gctx->gcm,
+ in +bulk,
+ out+bulk,
+ len-bulk))
return -1;
}
}
{
if (gctx->ctr)
{
+ size_t bulk=0;
+#if defined(AES_GCM_ASM)
+ if (len>=16 && AES_GCM_ASM(gctx))
+ {
+ size_t res = (16-gctx->gcm.mres)%16;
+
+ if (CRYPTO_gcm128_decrypt(&gctx->gcm,
+ in,out,res))
+ return -1;
+
+ bulk = AES_gcm_decrypt(in+res,
+ out+res,len-res,
+ gctx->gcm.key,
+ gctx->gcm.Yi.c,
+ gctx->gcm.Xi.u);
+ gctx->gcm.len.u[1] += bulk;
+ bulk += res;
+ }
+#endif
if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
- in, out, len,
+ in +bulk,
+ out+bulk,
+ len-bulk,
gctx->ctr))
return -1;
}
else {
- if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, len))
+ size_t bulk=0;
+#if defined(AES_GCM_ASM2)
+ if (len>=16 && AES_GCM_ASM2(gctx))
+ {
+ size_t res = (16-gctx->gcm.mres)%16;
+
+ if (CRYPTO_gcm128_decrypt(&gctx->gcm,
+ in,out,res))
+ return -1;
+
+ bulk = AES_gcm_decrypt(in+res,
+ out+res,len-res,
+ gctx->gcm.key,
+ gctx->gcm.Yi.c,
+ gctx->gcm.Xi.u);
+ gctx->gcm.len.u[1] += bulk;
+ bulk += res;
+ }
+#endif
+ if (CRYPTO_gcm128_decrypt(&gctx->gcm,
+ in +bulk,
+ out+bulk,
+ len-bulk))
return -1;
}
}
xctx->xts.block1 = (block128_f)vpaes_decrypt;
}
- vpaes_set_encrypt_key(key + ctx->key_len/2,
+ vpaes_set_encrypt_key(key + ctx->key_len/2,
ctx->key_len * 4, &xctx->ks2.ks);
- xctx->xts.block2 = (block128_f)vpaes_encrypt;
+ xctx->xts.block2 = (block128_f)vpaes_encrypt;
- xctx->xts.key1 = &xctx->ks1;
- break;
- }
+ xctx->xts.key1 = &xctx->ks1;
+ break;
+ }
+ else
#endif
+ (void)0; /* terminate potentially open 'else' */
+
if (enc)
{
AES_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
BLOCK_CIPHER_custom(NID_aes,192,1,12,ccm,CCM,EVP_CIPH_FLAG_FIPS|CUSTOM_FLAGS)
BLOCK_CIPHER_custom(NID_aes,256,1,12,ccm,CCM,EVP_CIPH_FLAG_FIPS|CUSTOM_FLAGS)
+typedef struct
+ {
+ union { double align; AES_KEY ks; } ks;
+ /* Indicates if IV has been set */
+ unsigned char *iv;
+ } EVP_AES_WRAP_CTX;
+
+static int aes_wrap_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
+ const unsigned char *iv, int enc)
+ {
+ EVP_AES_WRAP_CTX *wctx = ctx->cipher_data;
+ if (!iv && !key)
+ return 1;
+ if (key)
+ {
+ if (ctx->encrypt)
+ AES_set_encrypt_key(key, ctx->key_len * 8, &wctx->ks.ks);
+ else
+ AES_set_decrypt_key(key, ctx->key_len * 8, &wctx->ks.ks);
+ if (!iv)
+ wctx->iv = NULL;
+ }
+ if (iv)
+ {
+ memcpy(ctx->iv, iv, 8);
+ wctx->iv = ctx->iv;
+ }
+ return 1;
+ }
+
+static int aes_wrap_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
+ const unsigned char *in, size_t inlen)
+ {
+ EVP_AES_WRAP_CTX *wctx = ctx->cipher_data;
+ size_t rv;
+ if (inlen % 8)
+ return 0;
+ if (!out)
+ {
+ if (ctx->encrypt)
+ return inlen + 8;
+ else
+ return inlen - 8;
+ }
+ if (!in)
+ return 0;
+ if (ctx->encrypt)
+ rv = CRYPTO_128_wrap(&wctx->ks.ks, wctx->iv, out, in, inlen,
+ (block128_f)AES_encrypt);
+ else
+ rv = CRYPTO_128_unwrap(&wctx->ks.ks, wctx->iv, out, in, inlen,
+ (block128_f)AES_decrypt);
+ return rv ? (int)rv : -1;
+ }
+
+#define WRAP_FLAGS (EVP_CIPH_WRAP_MODE \
+ | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \
+ | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_FLAG_DEFAULT_ASN1)
+
+static const EVP_CIPHER aes_128_wrap = {
+ NID_id_aes128_wrap,
+ 8, 16, 8, WRAP_FLAGS,
+ aes_wrap_init_key, aes_wrap_cipher,
+ NULL,
+ sizeof(EVP_AES_WRAP_CTX),
+ NULL,NULL,NULL,NULL };
+
+const EVP_CIPHER *EVP_aes_128_wrap(void)
+ {
+ return &aes_128_wrap;
+ }
+
+static const EVP_CIPHER aes_192_wrap = {
+ NID_id_aes192_wrap,
+ 8, 24, 8, WRAP_FLAGS,
+ aes_wrap_init_key, aes_wrap_cipher,
+ NULL,
+ sizeof(EVP_AES_WRAP_CTX),
+ NULL,NULL,NULL,NULL };
+
+const EVP_CIPHER *EVP_aes_192_wrap(void)
+ {
+ return &aes_192_wrap;
+ }
+
+static const EVP_CIPHER aes_256_wrap = {
+ NID_id_aes256_wrap,
+ 8, 32, 8, WRAP_FLAGS,
+ aes_wrap_init_key, aes_wrap_cipher,
+ NULL,
+ sizeof(EVP_AES_WRAP_CTX),
+ NULL,NULL,NULL,NULL };
+
+const EVP_CIPHER *EVP_aes_256_wrap(void)
+ {
+ return &aes_256_wrap;
+ }
+
#endif