Remove old digest type hacks for non RSA keys.
[openssl.git] / crypto / pkcs7 / pk7_doit.c
index 8d98a85210c4abc9b86b29172a67a641b947e02c..60cffb0782c053245254c20efa1ec750c89759f6 100644 (file)
@@ -62,6 +62,7 @@
 #include <openssl/objects.h>
 #include <openssl/x509.h>
 #include <openssl/x509v3.h>
+#include <openssl/err.h>
 
 static int add_attribute(STACK_OF(X509_ATTRIBUTE) **sk, int nid, int atrtype,
                         void *value);
@@ -137,6 +138,121 @@ static int PKCS7_bio_add_digest(BIO **pbio, X509_ALGOR *alg)
 
        }
 
+static int pkcs7_encode_rinfo(PKCS7_RECIP_INFO *ri,
+                                       unsigned char *key, int keylen)
+       {
+       EVP_PKEY_CTX *pctx = NULL;
+       EVP_PKEY *pkey = NULL;
+       unsigned char *ek = NULL;
+       int ret = 0;
+       int eklen;
+
+       pkey = X509_get_pubkey(ri->cert);
+
+       if (!pkey)
+               return 0;
+
+       pctx = EVP_PKEY_CTX_new(pkey, NULL);
+       if (!pctx)
+               return 0;
+
+       if (EVP_PKEY_encrypt_init(pctx) <= 0)
+               goto err;
+
+       if (EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_ENCRYPT,
+                               EVP_PKEY_CTRL_PKCS7_ENCRYPT, 0, ri) <= 0)
+               {
+               PKCS7err(PKCS7_F_PKCS7_ENCODE_RINFO, PKCS7_R_CTRL_ERROR);
+               goto err;
+               }
+
+       if (EVP_PKEY_encrypt(pctx, NULL, &eklen, key, keylen) <= 0)
+               goto err;
+
+       ek = OPENSSL_malloc(eklen);
+
+       if (ek == NULL)
+               {
+               PKCS7err(PKCS7_F_PKCS7_ENCODE_RINFO, ERR_R_MALLOC_FAILURE);
+               goto err;
+               }
+
+       if (EVP_PKEY_encrypt(pctx, ek, &eklen, key, keylen) <= 0)
+               goto err;
+
+       ASN1_STRING_set0(ri->enc_key, ek, eklen);
+       ek = NULL;
+
+       ret = 1;
+
+       err:
+       if (pkey)
+               EVP_PKEY_free(pkey);
+       if (pctx)
+               EVP_PKEY_CTX_free(pctx);
+       if (ek)
+               OPENSSL_free(ek);
+       return ret;
+
+       }
+
+
+int pkcs7_decrypt_rinfo(unsigned char **pek, int *peklen,
+                       PKCS7_RECIP_INFO *ri, EVP_PKEY *pkey)
+       {
+       EVP_PKEY_CTX *pctx = NULL;
+       unsigned char *ek = NULL;
+       int eklen;
+
+       int ret = 0;
+
+       pctx = EVP_PKEY_CTX_new(pkey, NULL);
+       if (!pctx)
+               return 0;
+
+       if (EVP_PKEY_decrypt_init(pctx) <= 0)
+               goto err;
+
+       if (EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_DECRYPT,
+                               EVP_PKEY_CTRL_PKCS7_DECRYPT, 0, ri) <= 0)
+               {
+               PKCS7err(PKCS7_F_PKCS7_DECRYPT_RINFO, PKCS7_R_CTRL_ERROR);
+               goto err;
+               }
+
+       if (EVP_PKEY_decrypt(pctx, NULL, &eklen,
+                               ri->enc_key->data, ri->enc_key->length) <= 0)
+               goto err;
+
+       ek = OPENSSL_malloc(eklen);
+
+       if (ek == NULL)
+               {
+               PKCS7err(PKCS7_F_PKCS7_DECRYPT_RINFO, ERR_R_MALLOC_FAILURE);
+               goto err;
+               }
+
+       if (EVP_PKEY_decrypt(pctx, ek, &eklen,
+                               ri->enc_key->data, ri->enc_key->length) <= 0)
+               {
+               PKCS7err(PKCS7_F_PKCS7_DECRYPT_RINFO, ERR_R_EVP_LIB);
+               goto err;
+               }
+
+       ret = 1;
+
+       *pek = ek;
+       *peklen = eklen;
+
+       err:
+       if (pctx)
+               EVP_PKEY_CTX_free(pctx);
+       if (!ret && ek)
+               OPENSSL_free(ek);
+
+       return ret;
+       }
+
 BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
        {
        int i;
@@ -147,7 +263,6 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
        STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL;
        X509_ALGOR *xalg=NULL;
        PKCS7_RECIP_INFO *ri=NULL;
-       EVP_PKEY *pkey;
        ASN1_OCTET_STRING *os=NULL;
 
        i=OBJ_obj2nid(p7->type);
@@ -203,8 +318,6 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
                unsigned char key[EVP_MAX_KEY_LENGTH];
                unsigned char iv[EVP_MAX_IV_LENGTH];
                int keylen,ivlen;
-               int jj,max;
-               unsigned char *tmp;
                EVP_CIPHER_CTX *ctx;
 
                if ((btmp=BIO_new(BIO_f_cipher())) == NULL)
@@ -232,46 +345,12 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
                }
 
                /* Lets do the pub key stuff :-) */
-               max=0;
                for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
                        {
                        ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-                       if (ri->cert == NULL)
-                               {
-                               PKCS7err(PKCS7_F_PKCS7_DATAINIT,PKCS7_R_MISSING_CERIPEND_INFO);
+                       if (pkcs7_encode_rinfo(ri, key, keylen) <= 0)
                                goto err;
-                               }
-                       pkey=X509_get_pubkey(ri->cert);
-                       jj=EVP_PKEY_size(pkey);
-                       EVP_PKEY_free(pkey);
-                       if (max < jj) max=jj;
-                       }
-               if ((tmp=(unsigned char *)OPENSSL_malloc(max)) == NULL)
-                       {
-                       PKCS7err(PKCS7_F_PKCS7_DATAINIT,ERR_R_MALLOC_FAILURE);
-                       goto err;
                        }
-               for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
-                       {
-                       ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-                       pkey=X509_get_pubkey(ri->cert);
-                       jj=EVP_PKEY_encrypt(tmp,key,keylen,pkey);
-                       EVP_PKEY_free(pkey);
-                       if (jj <= 0)
-                               {
-                               PKCS7err(PKCS7_F_PKCS7_DATAINIT,ERR_R_EVP_LIB);
-                               OPENSSL_free(tmp);
-                               goto err;
-                               }
-                       if (!M_ASN1_OCTET_STRING_set(ri->enc_key,tmp,jj))
-                               {
-                               PKCS7err(PKCS7_F_PKCS7_DATAINIT,
-                                       ERR_R_MALLOC_FAILURE);
-                               OPENSSL_free(tmp);
-                               goto err;
-                               }
-                       }
-               OPENSSL_free(tmp);
                OPENSSL_cleanse(key, keylen);
 
                if (out == NULL)
@@ -307,12 +386,22 @@ err:
        return(out);
        }
 
+static int pkcs7_cmp_ri(PKCS7_RECIP_INFO *ri, X509 *pcert)
+       {
+       int ret;
+       ret = X509_NAME_cmp(ri->issuer_and_serial->issuer,
+                               pcert->cert_info->issuer);
+       if (ret)
+               return ret;
+       return M_ASN1_INTEGER_cmp(pcert->cert_info->serialNumber,
+                                       ri->issuer_and_serial->serial);
+       }
+
 /* int */
 BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
        {
        int i,j;
        BIO *out=NULL,*btmp=NULL,*etmp=NULL,*bio=NULL;
-       unsigned char *tmp=NULL;
        X509_ALGOR *xa;
        ASN1_OCTET_STRING *data_body=NULL;
        const EVP_MD *evp_md;
@@ -402,7 +491,8 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                int max;
                X509_OBJECT ret;
 #endif
-               int jj;
+               unsigned char *ek = NULL;
+               int eklen;
 
                if ((etmp=BIO_new(BIO_f_cipher())) == NULL)
                        {
@@ -417,34 +507,47 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                 * (if any)
                 */
 
-               for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++) {
-                       ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-                       if(!X509_NAME_cmp(ri->issuer_and_serial->issuer,
-                                       pcert->cert_info->issuer) &&
-                            !M_ASN1_INTEGER_cmp(pcert->cert_info->serialNumber,
-                                       ri->issuer_and_serial->serial)) break;
-                       ri=NULL;
-               }
-               if (ri == NULL) {
-                       PKCS7err(PKCS7_F_PKCS7_DATADECODE,
-                                PKCS7_R_NO_RECIPIENT_MATCHES_CERTIFICATE);
-                       goto err;
-               }
-
-               jj=EVP_PKEY_size(pkey);
-               tmp=(unsigned char *)OPENSSL_malloc(jj+10);
-               if (tmp == NULL)
+               if (pcert)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_MALLOC_FAILURE);
-                       goto err;
+                       for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
+                               {
+                               ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
+                               if (!pkcs7_cmp_ri(ri, pcert))
+                                       break;
+                               ri=NULL;
+                               }
+                       if (ri == NULL)
+                               {
+                               PKCS7err(PKCS7_F_PKCS7_DATADECODE,
+                                     PKCS7_R_NO_RECIPIENT_MATCHES_CERTIFICATE);
+                               goto err;
+                               }
                        }
 
-               jj=EVP_PKEY_decrypt(tmp, M_ASN1_STRING_data(ri->enc_key),
-                       M_ASN1_STRING_length(ri->enc_key), pkey);
-               if (jj <= 0)
+               /* If we haven't got a certificate try each ri in turn */
+
+               if (pcert == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_EVP_LIB);
-                       goto err;
+                       for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
+                               {
+                               ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
+                               if (pkcs7_decrypt_rinfo(&ek, &eklen,
+                                                       ri, pkey) > 0)
+                                       break;
+                               ERR_clear_error();
+                               ri = NULL;
+                               }
+                       if (ri == NULL)
+                               {
+                               PKCS7err(PKCS7_F_PKCS7_DATADECODE,
+                                     PKCS7_R_NO_RECIPIENT_MATCHES_KEY);
+                               goto err;
+                               }
+                       }
+               else
+                       {
+                       if (pkcs7_decrypt_rinfo(&ek, &eklen, ri, pkey) <= 0)
+                               goto err;
                        }
 
                evp_ctx=NULL;
@@ -454,22 +557,26 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                if (EVP_CIPHER_asn1_to_param(evp_ctx,enc_alg->parameter) < 0)
                        goto err;
 
-               if (jj != EVP_CIPHER_CTX_key_length(evp_ctx)) {
+               if (eklen != EVP_CIPHER_CTX_key_length(evp_ctx)) {
                        /* Some S/MIME clients don't use the same key
                         * and effective key length. The key length is
                         * determined by the size of the decrypted RSA key.
                         */
-                       if(!EVP_CIPHER_CTX_set_key_length(evp_ctx, jj))
+                       if(!EVP_CIPHER_CTX_set_key_length(evp_ctx, eklen))
                                {
                                PKCS7err(PKCS7_F_PKCS7_DATADECODE,
                                        PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH);
                                goto err;
                                }
                } 
-               if (EVP_CipherInit_ex(evp_ctx,NULL,NULL,tmp,NULL,0) <= 0)
+               if (EVP_CipherInit_ex(evp_ctx,NULL,NULL,ek,NULL,0) <= 0)
                        goto err;
 
-               OPENSSL_cleanse(tmp,jj);
+               if (ek)
+                       {
+                       OPENSSL_cleanse(ek,eklen);
+                       OPENSSL_free(ek);
+                       }
 
                if (out == NULL)
                        out=etmp;
@@ -515,8 +622,6 @@ err:
                if (bio != NULL) BIO_free_all(bio);
                out=NULL;
                }
-       if (tmp != NULL)
-               OPENSSL_free(tmp);
        return(out);
        }
 
@@ -527,13 +632,13 @@ static BIO *PKCS7_find_digest(EVP_MD_CTX **pmd, BIO *bio, int nid)
                bio=BIO_find_type(bio,BIO_TYPE_MD);
                if (bio == NULL)
                        {
-                       PKCS7err(PKCS7_F_FIND_DIGEST,PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST);
+                       PKCS7err(PKCS7_F_PKCS7_FIND_DIGEST,PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST);
                        return NULL;    
                        }
                BIO_get_md_ctx(bio,pmd);
                if (*pmd == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_INTERNAL_ERROR);
+                       PKCS7err(PKCS7_F_PKCS7_FIND_DIGEST,ERR_R_INTERNAL_ERROR);
                        return NULL;
                        }       
                if (EVP_MD_CTX_type(*pmd) == nid)
@@ -567,7 +672,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                si_sk=p7->d.signed_and_enveloped->signer_info;
                if (!(os=M_ASN1_OCTET_STRING_new()))
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_MALLOC_FAILURE);
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_MALLOC_FAILURE);
                        goto err;
                        }
                p7->d.signed_and_enveloped->enc_data->enc_data=os;
@@ -576,7 +681,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                /* XXXXXXXXXXXXXXXX */
                if (!(os=M_ASN1_OCTET_STRING_new()))
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_MALLOC_FAILURE);
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_MALLOC_FAILURE);
                        goto err;
                        }
                p7->d.enveloped->enc_data->enc_data=os;
@@ -607,7 +712,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                {
                if ((buf=BUF_MEM_new()) == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_BIO_LIB);
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_BIO_LIB);
                        goto err;
                        }
                for (i=0; i<sk_PKCS7_SIGNER_INFO_num(si_sk); i++)
@@ -629,7 +734,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                        EVP_MD_CTX_copy_ex(&ctx_tmp,mdc);
                        if (!BUF_MEM_grow_clean(buf,EVP_PKEY_size(si->pkey)))
                                {
-                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_BIO_LIB);
+                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_BIO_LIB);
                                goto err;
                                }
 
@@ -651,7 +756,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                                        {
                                        if (!(sign_time=X509_gmtime_adj(NULL,0)))
                                                {
-                                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,
+                                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,
                                                        ERR_R_MALLOC_FAILURE);
                                                goto err;
                                                }
@@ -665,14 +770,14 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                                EVP_DigestFinal_ex(&ctx_tmp,md_data,&md_len);
                                if (!(digest=M_ASN1_OCTET_STRING_new()))
                                        {
-                                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,
+                                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,
                                                ERR_R_MALLOC_FAILURE);
                                        goto err;
                                        }
                                if (!M_ASN1_OCTET_STRING_set(digest,md_data,
                                                                md_len))
                                        {
-                                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,
+                                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,
                                                ERR_R_MALLOC_FAILURE);
                                        goto err;
                                        }
@@ -689,25 +794,16 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                                OPENSSL_free(abuf);
                                }
 
-#ifndef OPENSSL_NO_DSA
-                       if (si->pkey->type == EVP_PKEY_DSA)
-                               ctx_tmp.digest=EVP_dss1();
-#endif
-#ifndef OPENSSL_NO_ECDSA
-                       if (si->pkey->type == EVP_PKEY_EC)
-                               ctx_tmp.digest=EVP_ecdsa();
-#endif
-
                        if (!EVP_SignFinal(&ctx_tmp,(unsigned char *)buf->data,
                                (unsigned int *)&buf->length,si->pkey))
                                {
-                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_EVP_LIB);
+                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_EVP_LIB);
                                goto err;
                                }
                        if (!ASN1_STRING_set(si->enc_digest,
                                (unsigned char *)buf->data,buf->length))
                                {
-                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_ASN1_LIB);
+                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_ASN1_LIB);
                                goto err;
                                }
                        }
@@ -728,7 +824,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                btmp=BIO_find_type(bio,BIO_TYPE_MEM);
                if (btmp == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,PKCS7_R_UNABLE_TO_FIND_MEM_BIO);
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,PKCS7_R_UNABLE_TO_FIND_MEM_BIO);
                        goto err;
                        }
                BIO_get_mem_ptr(btmp,&buf_mem);
@@ -906,12 +1002,6 @@ for (ii=0; ii<md_len; ii++) printf("%02X",md_dat[ii]); printf(" calc\n");
                ret = -1;
                goto err;
                }
-#ifndef OPENSSL_NO_DSA
-       if(pkey->type == EVP_PKEY_DSA) mdc_tmp.digest=EVP_dss1();
-#endif
-#ifndef OPENSSL_NO_ECDSA
-       if (pkey->type == EVP_PKEY_EC) mdc_tmp.digest=EVP_ecdsa();
-#endif
 
        i=EVP_VerifyFinal(&mdc_tmp,os->data,os->length, pkey);
        EVP_PKEY_free(pkey);