Fix to PKCS#7 routines so it can decrypt some oddball RC2 handling.

diff --git a/CHANGES b/CHANGES
index 1c4ce47afbda02500654cdad7f2fc3c25cacbbcb..27a6a80ba67dee5239809a07b1455a35b515b32a 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -4,6 +4,18 @@
 
  Changes between 0.9.4 and 0.9.5  [xx XXX 1999]
 
+  *) Hack to fix PKCS#7 decryption when used with some unorthodox RC2 
+     handling. Most clients have the effective key size in bits equal to
+     the key length in bits: so a 40 bit RC2 key uses a 40 bit (5 byte) key.
+     A few however don't do this and instead use the size of the decrypted key
+     to determine the RC2 key length and the AlgorithmIdentifier to determine
+     the effective key length. In this case the effective key lenth can still
+     be 40 bits but the key length can be 168 bits for example. This is fixed
+     by manually forcing an RC2 key into the EVP_PKEY structure because the
+     EVP code can't currently handle unusual RC2 key sizes: it always assumes
+     the key length and effective key length are equal.
+     [Steve Henson]
+
   *) Add a bunch of functions that should simplify the creation of 
      X509_NAME structures. Now you should be able to do:
      X509_NAME_add_entry_by_txt(nm, "CN", MBSTRING_ASC, "Steve", -1, -1, 0);

diff --git a/crypto/pkcs7/pk7_doit.c b/crypto/pkcs7/pk7_doit.c
index a4cf9b4021df1209064f12571d9a0b7b8dc8884b..4c32f053b7e0cfaaac8306c141e42b3d75f42ce9 100644 (file)
--- a/crypto/pkcs7/pk7_doit.c
+++ b/crypto/pkcs7/pk7_doit.c
@@ -251,7 +251,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
        {
        int i,j;
        BIO *out=NULL,*btmp=NULL,*etmp=NULL,*bio=NULL;
-       char *tmp=NULL;
+       unsigned char *tmp=NULL;
        X509_ALGOR *xa;
        ASN1_OCTET_STRING *data_body=NULL;
        const EVP_MD *evp_md;
@@ -262,6 +262,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
        STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL;
        X509_ALGOR *xalg=NULL;
        PKCS7_RECIP_INFO *ri=NULL;
+       char is_rc2 = 0;
 /*     EVP_PKEY *pkey; */
 #if 0
        X509_STORE_CTX s_ctx;
@@ -306,6 +307,8 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                goto err;
                }
 
+       if(EVP_CIPHER_nid(evp_cipher) == NID_rc2_cbc) is_rc2 = 1; 
+
        /* We will be checking the signature */
        if (md_sk != NULL)
                {
@@ -375,17 +378,15 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                }
 
                jj=EVP_PKEY_size(pkey);
-               tmp=Malloc(jj+10);
+               tmp=(unsigned char *)Malloc(jj+10);
                if (tmp == NULL)
                        {
                        PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_MALLOC_FAILURE);
                        goto err;
                        }
 
-               jj=EVP_PKEY_decrypt((unsigned char *)tmp,
-                       M_ASN1_STRING_data(ri->enc_key),
-                       M_ASN1_STRING_length(ri->enc_key),
-                       pkey);
+               jj=EVP_PKEY_decrypt(tmp, M_ASN1_STRING_data(ri->enc_key),
+                       M_ASN1_STRING_length(ri->enc_key), pkey);
                if (jj <= 0)
                        {
                        PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_EVP_LIB);
@@ -398,13 +399,23 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                if (EVP_CIPHER_asn1_to_param(evp_ctx,enc_alg->parameter) < 0)
                        return(NULL);
 
-               if (jj != EVP_CIPHER_CTX_key_length(evp_ctx))
-                       {
-                       PKCS7err(PKCS7_F_PKCS7_DATADECODE,
+               if (jj != EVP_CIPHER_CTX_key_length(evp_ctx)) {
+                       /* HACK: some S/MIME clients don't use the same key
+                        * and effective key length. The key length is
+                        * determined by the size of the decrypted RSA key.
+                        * So we hack things to manually set the RC2 key
+                        * because we currently can't do this with the EVP
+                        * interface.
+                        */
+                       if(is_rc2) RC2_set_key(&(evp_ctx->c.rc2_ks),jj, tmp,
+                                       EVP_CIPHER_CTX_key_length(evp_ctx)*8);
+                       else {
+
+                               PKCS7err(PKCS7_F_PKCS7_DATADECODE,
                                        PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH);
-                       goto err;
+                               goto err;
                        }
-               EVP_CipherInit(evp_ctx,NULL,(unsigned char *)tmp,NULL,0);
+               } else EVP_CipherInit(evp_ctx,NULL,tmp,NULL,0);
 
                memset(tmp,0,jj);