Remove dss1 hack from S/MIME code.

[openssl.git] / crypto / pkcs7 / pk7_doit.c

diff --git a/crypto/pkcs7/pk7_doit.c b/crypto/pkcs7/pk7_doit.c
index 8d98a85210c4abc9b86b29172a67a641b947e02c..67bb8264e02b7d77a3df6ecec20cf2fdbc96a88a 100644 (file)
--- a/crypto/pkcs7/pk7_doit.c
+++ b/crypto/pkcs7/pk7_doit.c
@@ -62,6 +62,7 @@
 #include <openssl/objects.h>
 #include <openssl/x509.h>
 #include <openssl/x509v3.h>
+#include <openssl/err.h>
 
 static int add_attribute(STACK_OF(X509_ATTRIBUTE) **sk, int nid, int atrtype,
                         void *value);
@@ -255,7 +256,7 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
                        {
                        ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
                        pkey=X509_get_pubkey(ri->cert);
-                       jj=EVP_PKEY_encrypt(tmp,key,keylen,pkey);
+                       jj=EVP_PKEY_encrypt_old(tmp,key,keylen,pkey);
                        EVP_PKEY_free(pkey);
                        if (jj <= 0)
                                {
@@ -307,6 +308,17 @@ err:
        return(out);
        }
 
+static int pkcs7_cmp_ri(PKCS7_RECIP_INFO *ri, X509 *pcert)
+       {
+       int ret;
+       ret = X509_NAME_cmp(ri->issuer_and_serial->issuer,
+                               pcert->cert_info->issuer);
+       if (ret)
+               return ret;
+       return M_ASN1_INTEGER_cmp(pcert->cert_info->serialNumber,
+                                       ri->issuer_and_serial->serial);
+       }
+
 /* int */
 BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
        {
@@ -417,18 +429,18 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                 * (if any)
                 */
 
-               for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++) {
-                       ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-                       if(!X509_NAME_cmp(ri->issuer_and_serial->issuer,
-                                       pcert->cert_info->issuer) &&
-                            !M_ASN1_INTEGER_cmp(pcert->cert_info->serialNumber,
-                                       ri->issuer_and_serial->serial)) break;
-                       ri=NULL;
-               }
-               if (ri == NULL) {
-                       PKCS7err(PKCS7_F_PKCS7_DATADECODE,
-                                PKCS7_R_NO_RECIPIENT_MATCHES_CERTIFICATE);
-                       goto err;
+               if (pcert) {
+                       for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++) {
+                               ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
+                               if (!pkcs7_cmp_ri(ri, pcert))
+                                       break;
+                               ri=NULL;
+                       }
+                       if (ri == NULL) {
+                               PKCS7err(PKCS7_F_PKCS7_DATADECODE,
+                                     PKCS7_R_NO_RECIPIENT_MATCHES_CERTIFICATE);
+                               goto err;
+                       }
                }
 
                jj=EVP_PKEY_size(pkey);
@@ -439,12 +451,40 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                        goto err;
                        }
 
-               jj=EVP_PKEY_decrypt(tmp, M_ASN1_STRING_data(ri->enc_key),
-                       M_ASN1_STRING_length(ri->enc_key), pkey);
-               if (jj <= 0)
+               /* If we haven't got a certificate try each ri in turn */
+
+               if (pcert == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_EVP_LIB);
-                       goto err;
+                       for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
+                               {
+                               ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
+                               jj=EVP_PKEY_decrypt_old(tmp,
+                                       M_ASN1_STRING_data(ri->enc_key),
+                                       M_ASN1_STRING_length(ri->enc_key),
+                                               pkey);
+                               if (jj > 0)
+                                       break;
+                               ERR_clear_error();
+                               ri = NULL;
+                               }
+                       if (ri == NULL)
+                               {
+                               PKCS7err(PKCS7_F_PKCS7_DATADECODE,
+                                     PKCS7_R_NO_RECIPIENT_MATCHES_KEY);
+                               goto err;
+                               }
+                       }
+               else
+                       {
+                       jj=EVP_PKEY_decrypt_old(tmp,
+                               M_ASN1_STRING_data(ri->enc_key),
+                               M_ASN1_STRING_length(ri->enc_key), pkey);
+                       if (jj <= 0)
+                               {
+                               PKCS7err(PKCS7_F_PKCS7_DATADECODE,
+                                                               ERR_R_EVP_LIB);
+                               goto err;
+                               }
                        }
 
                evp_ctx=NULL;
@@ -527,13 +567,13 @@ static BIO *PKCS7_find_digest(EVP_MD_CTX **pmd, BIO *bio, int nid)
                bio=BIO_find_type(bio,BIO_TYPE_MD);
                if (bio == NULL)
                        {
-                       PKCS7err(PKCS7_F_FIND_DIGEST,PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST);
+                       PKCS7err(PKCS7_F_PKCS7_FIND_DIGEST,PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST);
                        return NULL;    
                        }
                BIO_get_md_ctx(bio,pmd);
                if (*pmd == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_INTERNAL_ERROR);
+                       PKCS7err(PKCS7_F_PKCS7_FIND_DIGEST,ERR_R_INTERNAL_ERROR);
                        return NULL;
                        }       
                if (EVP_MD_CTX_type(*pmd) == nid)
@@ -567,7 +607,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                si_sk=p7->d.signed_and_enveloped->signer_info;
                if (!(os=M_ASN1_OCTET_STRING_new()))
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_MALLOC_FAILURE);
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_MALLOC_FAILURE);
                        goto err;
                        }
                p7->d.signed_and_enveloped->enc_data->enc_data=os;
@@ -576,7 +616,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                /* XXXXXXXXXXXXXXXX */
                if (!(os=M_ASN1_OCTET_STRING_new()))
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_MALLOC_FAILURE);
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_MALLOC_FAILURE);
                        goto err;
                        }
                p7->d.enveloped->enc_data->enc_data=os;
@@ -607,7 +647,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                {
                if ((buf=BUF_MEM_new()) == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_BIO_LIB);
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_BIO_LIB);
                        goto err;
                        }
                for (i=0; i<sk_PKCS7_SIGNER_INFO_num(si_sk); i++)
@@ -629,7 +669,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                        EVP_MD_CTX_copy_ex(&ctx_tmp,mdc);
                        if (!BUF_MEM_grow_clean(buf,EVP_PKEY_size(si->pkey)))
                                {
-                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_BIO_LIB);
+                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_BIO_LIB);
                                goto err;
                                }
 
@@ -651,7 +691,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                                        {
                                        if (!(sign_time=X509_gmtime_adj(NULL,0)))
                                                {
-                                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,
+                                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,
                                                        ERR_R_MALLOC_FAILURE);
                                                goto err;
                                                }
@@ -665,14 +705,14 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                                EVP_DigestFinal_ex(&ctx_tmp,md_data,&md_len);
                                if (!(digest=M_ASN1_OCTET_STRING_new()))
                                        {
-                                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,
+                                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,
                                                ERR_R_MALLOC_FAILURE);
                                        goto err;
                                        }
                                if (!M_ASN1_OCTET_STRING_set(digest,md_data,
                                                                md_len))
                                        {
-                                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,
+                                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,
                                                ERR_R_MALLOC_FAILURE);
                                        goto err;
                                        }
@@ -701,13 +741,13 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                        if (!EVP_SignFinal(&ctx_tmp,(unsigned char *)buf->data,
                                (unsigned int *)&buf->length,si->pkey))
                                {
-                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_EVP_LIB);
+                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_EVP_LIB);
                                goto err;
                                }
                        if (!ASN1_STRING_set(si->enc_digest,
                                (unsigned char *)buf->data,buf->length))
                                {
-                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_ASN1_LIB);
+                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_ASN1_LIB);
                                goto err;
                                }
                        }
@@ -728,7 +768,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                btmp=BIO_find_type(bio,BIO_TYPE_MEM);
                if (btmp == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,PKCS7_R_UNABLE_TO_FIND_MEM_BIO);
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,PKCS7_R_UNABLE_TO_FIND_MEM_BIO);
                        goto err;
                        }
                BIO_get_mem_ptr(btmp,&buf_mem);
@@ -906,11 +946,13 @@ for (ii=0; ii<md_len; ii++) printf("%02X",md_dat[ii]); printf(" calc\n");
                ret = -1;
                goto err;
                }
+#if 0
 #ifndef OPENSSL_NO_DSA
        if(pkey->type == EVP_PKEY_DSA) mdc_tmp.digest=EVP_dss1();
 #endif
 #ifndef OPENSSL_NO_ECDSA
        if (pkey->type == EVP_PKEY_EC) mdc_tmp.digest=EVP_ecdsa();
+#endif
 #endif
 
        i=EVP_VerifyFinal(&mdc_tmp,os->data,os->length, pkey);