Restore EVP_CIPH_FLAG_LENGTH_BITS working properly
authorLukasz Pawelczyk <havner@gmail.com>
Thu, 17 Nov 2016 09:31:39 +0000 (10:31 +0100)
committerMatt Caswell <matt@openssl.org>
Fri, 10 Feb 2017 13:36:56 +0000 (13:36 +0000)
EVP_CIPH_FLAG_LENGTH_BITS flag for CFB1 has been broken with the
introduction of the is_partially_overlapping() check that did not take
it into the account (treating number of bits passed as bytes). This
remedies that and allows this flag to work as intended.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/1942)

crypto/evp/evp_enc.c

index bce2b57..f829e8d 100644 (file)
@@ -295,13 +295,16 @@ int is_partially_overlapping(const void *ptr1, const void *ptr2, int len)
 int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                       const unsigned char *in, int inl)
 {
-    int i, j, bl;
+    int i, j, bl, cmpl = inl;
+
+    if (EVP_CIPHER_CTX_test_flags(ctx, EVP_CIPH_FLAG_LENGTH_BITS))
+        cmpl = (cmpl + 7) / 8;
 
     bl = ctx->cipher->block_size;
 
     if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) {
         /* If block size > 1 then the cipher will have to do this check */
-        if (bl == 1 && is_partially_overlapping(out, in, inl)) {
+        if (bl == 1 && is_partially_overlapping(out, in, cmpl)) {
             EVPerr(EVP_F_EVP_ENCRYPTUPDATE, EVP_R_PARTIALLY_OVERLAPPING);
             return 0;
         }
@@ -318,7 +321,7 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
         *outl = 0;
         return inl == 0;
     }
-    if (is_partially_overlapping(out + ctx->buf_len, in, inl)) {
+    if (is_partially_overlapping(out + ctx->buf_len, in, cmpl)) {
         EVPerr(EVP_F_EVP_ENCRYPTUPDATE, EVP_R_PARTIALLY_OVERLAPPING);
         return 0;
     }
@@ -418,13 +421,16 @@ int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
 int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                       const unsigned char *in, int inl)
 {
-    int fix_len;
+    int fix_len, cmpl = inl;
     unsigned int b;
 
     b = ctx->cipher->block_size;
 
+    if (EVP_CIPHER_CTX_test_flags(ctx, EVP_CIPH_FLAG_LENGTH_BITS))
+        cmpl = (cmpl + 7) / 8;
+
     if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) {
-        if (b == 1 && is_partially_overlapping(out, in, inl)) {
+        if (b == 1 && is_partially_overlapping(out, in, cmpl)) {
             EVPerr(EVP_F_EVP_DECRYPTUPDATE, EVP_R_PARTIALLY_OVERLAPPING);
             return 0;
         }