From 9716a8f9f2cc3cb78bfa6229a608badf24aef1ef Mon Sep 17 00:00:00 2001 From: "Dr. Stephen Henson" Date: Fri, 29 Oct 1999 13:06:25 +0000 Subject: [PATCH] Fix to PKCS#7 routines so it can decrypt some oddball RC2 handling. --- CHANGES | 12 ++++++++++++ crypto/pkcs7/pk7_doit.c | 33 ++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/CHANGES b/CHANGES index 1c4ce47afb..27a6a80ba6 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,18 @@ Changes between 0.9.4 and 0.9.5 [xx XXX 1999] + *) Hack to fix PKCS#7 decryption when used with some unorthodox RC2 + handling. Most clients have the effective key size in bits equal to + the key length in bits: so a 40 bit RC2 key uses a 40 bit (5 byte) key. + A few however don't do this and instead use the size of the decrypted key + to determine the RC2 key length and the AlgorithmIdentifier to determine + the effective key length. In this case the effective key lenth can still + be 40 bits but the key length can be 168 bits for example. This is fixed + by manually forcing an RC2 key into the EVP_PKEY structure because the + EVP code can't currently handle unusual RC2 key sizes: it always assumes + the key length and effective key length are equal. + [Steve Henson] + *) Add a bunch of functions that should simplify the creation of X509_NAME structures. Now you should be able to do: X509_NAME_add_entry_by_txt(nm, "CN", MBSTRING_ASC, "Steve", -1, -1, 0); diff --git a/crypto/pkcs7/pk7_doit.c b/crypto/pkcs7/pk7_doit.c index a4cf9b4021..4c32f053b7 100644 --- a/crypto/pkcs7/pk7_doit.c +++ b/crypto/pkcs7/pk7_doit.c @@ -251,7 +251,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) { int i,j; BIO *out=NULL,*btmp=NULL,*etmp=NULL,*bio=NULL; - char *tmp=NULL; + unsigned char *tmp=NULL; X509_ALGOR *xa; ASN1_OCTET_STRING *data_body=NULL; const EVP_MD *evp_md; @@ -262,6 +262,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL; X509_ALGOR *xalg=NULL; PKCS7_RECIP_INFO *ri=NULL; + char is_rc2 = 0; /* EVP_PKEY *pkey; */ #if 0 X509_STORE_CTX s_ctx; @@ -306,6 +307,8 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) goto err; } + if(EVP_CIPHER_nid(evp_cipher) == NID_rc2_cbc) is_rc2 = 1; + /* We will be checking the signature */ if (md_sk != NULL) { @@ -375,17 +378,15 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) } jj=EVP_PKEY_size(pkey); - tmp=Malloc(jj+10); + tmp=(unsigned char *)Malloc(jj+10); if (tmp == NULL) { PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_MALLOC_FAILURE); goto err; } - jj=EVP_PKEY_decrypt((unsigned char *)tmp, - M_ASN1_STRING_data(ri->enc_key), - M_ASN1_STRING_length(ri->enc_key), - pkey); + jj=EVP_PKEY_decrypt(tmp, M_ASN1_STRING_data(ri->enc_key), + M_ASN1_STRING_length(ri->enc_key), pkey); if (jj <= 0) { PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_EVP_LIB); @@ -398,13 +399,23 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) if (EVP_CIPHER_asn1_to_param(evp_ctx,enc_alg->parameter) < 0) return(NULL); - if (jj != EVP_CIPHER_CTX_key_length(evp_ctx)) - { - PKCS7err(PKCS7_F_PKCS7_DATADECODE, + if (jj != EVP_CIPHER_CTX_key_length(evp_ctx)) { + /* HACK: some S/MIME clients don't use the same key + * and effective key length. The key length is + * determined by the size of the decrypted RSA key. + * So we hack things to manually set the RC2 key + * because we currently can't do this with the EVP + * interface. + */ + if(is_rc2) RC2_set_key(&(evp_ctx->c.rc2_ks),jj, tmp, + EVP_CIPHER_CTX_key_length(evp_ctx)*8); + else { + + PKCS7err(PKCS7_F_PKCS7_DATADECODE, PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH); - goto err; + goto err; } - EVP_CipherInit(evp_ctx,NULL,(unsigned char *)tmp,NULL,0); + } else EVP_CipherInit(evp_ctx,NULL,tmp,NULL,0); memset(tmp,0,jj); -- 2.34.1