Remove old digest type hacks for non RSA keys.
[openssl.git] / crypto / pkcs7 / pk7_doit.c
index 0b262fa0653f6c1542fc8e8cdea6138b0f46cb8d..60cffb0782c053245254c20efa1ec750c89759f6 100644 (file)
@@ -62,6 +62,7 @@
 #include <openssl/objects.h>
 #include <openssl/x509.h>
 #include <openssl/x509v3.h>
+#include <openssl/err.h>
 
 static int add_attribute(STACK_OF(X509_ATTRIBUTE) **sk, int nid, int atrtype,
                         void *value);
@@ -137,6 +138,121 @@ static int PKCS7_bio_add_digest(BIO **pbio, X509_ALGOR *alg)
 
        }
 
+static int pkcs7_encode_rinfo(PKCS7_RECIP_INFO *ri,
+                                       unsigned char *key, int keylen)
+       {
+       EVP_PKEY_CTX *pctx = NULL;
+       EVP_PKEY *pkey = NULL;
+       unsigned char *ek = NULL;
+       int ret = 0;
+       int eklen;
+
+       pkey = X509_get_pubkey(ri->cert);
+
+       if (!pkey)
+               return 0;
+
+       pctx = EVP_PKEY_CTX_new(pkey, NULL);
+       if (!pctx)
+               return 0;
+
+       if (EVP_PKEY_encrypt_init(pctx) <= 0)
+               goto err;
+
+       if (EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_ENCRYPT,
+                               EVP_PKEY_CTRL_PKCS7_ENCRYPT, 0, ri) <= 0)
+               {
+               PKCS7err(PKCS7_F_PKCS7_ENCODE_RINFO, PKCS7_R_CTRL_ERROR);
+               goto err;
+               }
+
+       if (EVP_PKEY_encrypt(pctx, NULL, &eklen, key, keylen) <= 0)
+               goto err;
+
+       ek = OPENSSL_malloc(eklen);
+
+       if (ek == NULL)
+               {
+               PKCS7err(PKCS7_F_PKCS7_ENCODE_RINFO, ERR_R_MALLOC_FAILURE);
+               goto err;
+               }
+
+       if (EVP_PKEY_encrypt(pctx, ek, &eklen, key, keylen) <= 0)
+               goto err;
+
+       ASN1_STRING_set0(ri->enc_key, ek, eklen);
+       ek = NULL;
+
+       ret = 1;
+
+       err:
+       if (pkey)
+               EVP_PKEY_free(pkey);
+       if (pctx)
+               EVP_PKEY_CTX_free(pctx);
+       if (ek)
+               OPENSSL_free(ek);
+       return ret;
+
+       }
+
+
+int pkcs7_decrypt_rinfo(unsigned char **pek, int *peklen,
+                       PKCS7_RECIP_INFO *ri, EVP_PKEY *pkey)
+       {
+       EVP_PKEY_CTX *pctx = NULL;
+       unsigned char *ek = NULL;
+       int eklen;
+
+       int ret = 0;
+
+       pctx = EVP_PKEY_CTX_new(pkey, NULL);
+       if (!pctx)
+               return 0;
+
+       if (EVP_PKEY_decrypt_init(pctx) <= 0)
+               goto err;
+
+       if (EVP_PKEY_CTX_ctrl(pctx, -1, EVP_PKEY_OP_DECRYPT,
+                               EVP_PKEY_CTRL_PKCS7_DECRYPT, 0, ri) <= 0)
+               {
+               PKCS7err(PKCS7_F_PKCS7_DECRYPT_RINFO, PKCS7_R_CTRL_ERROR);
+               goto err;
+               }
+
+       if (EVP_PKEY_decrypt(pctx, NULL, &eklen,
+                               ri->enc_key->data, ri->enc_key->length) <= 0)
+               goto err;
+
+       ek = OPENSSL_malloc(eklen);
+
+       if (ek == NULL)
+               {
+               PKCS7err(PKCS7_F_PKCS7_DECRYPT_RINFO, ERR_R_MALLOC_FAILURE);
+               goto err;
+               }
+
+       if (EVP_PKEY_decrypt(pctx, ek, &eklen,
+                               ri->enc_key->data, ri->enc_key->length) <= 0)
+               {
+               PKCS7err(PKCS7_F_PKCS7_DECRYPT_RINFO, ERR_R_EVP_LIB);
+               goto err;
+               }
+
+       ret = 1;
+
+       *pek = ek;
+       *peklen = eklen;
+
+       err:
+       if (pctx)
+               EVP_PKEY_CTX_free(pctx);
+       if (!ret && ek)
+               OPENSSL_free(ek);
+
+       return ret;
+       }
+
 BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
        {
        int i;
@@ -147,7 +263,6 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
        STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL;
        X509_ALGOR *xalg=NULL;
        PKCS7_RECIP_INFO *ri=NULL;
-       EVP_PKEY *pkey;
        ASN1_OCTET_STRING *os=NULL;
 
        i=OBJ_obj2nid(p7->type);
@@ -203,8 +318,6 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
                unsigned char key[EVP_MAX_KEY_LENGTH];
                unsigned char iv[EVP_MAX_IV_LENGTH];
                int keylen,ivlen;
-               int jj,max;
-               unsigned char *tmp;
                EVP_CIPHER_CTX *ctx;
 
                if ((btmp=BIO_new(BIO_f_cipher())) == NULL)
@@ -215,11 +328,14 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
                BIO_get_cipher_ctx(btmp, &ctx);
                keylen=EVP_CIPHER_key_length(evp_cipher);
                ivlen=EVP_CIPHER_iv_length(evp_cipher);
-               if (RAND_bytes(key,keylen) <= 0)
-                       goto err;
                xalg->algorithm = OBJ_nid2obj(EVP_CIPHER_type(evp_cipher));
                if (ivlen > 0) RAND_pseudo_bytes(iv,ivlen);
-               EVP_CipherInit_ex(ctx, evp_cipher, NULL, key, iv, 1);
+               if (EVP_CipherInit_ex(ctx, evp_cipher, NULL, NULL, NULL, 1)<=0)
+                       goto err;
+               if (EVP_CIPHER_CTX_rand_key(ctx, key) <= 0)
+                       goto err;
+               if (EVP_CipherInit_ex(ctx, NULL, NULL, key, iv, 1) <= 0)
+                       goto err;
 
                if (ivlen > 0) {
                        if (xalg->parameter == NULL) 
@@ -229,40 +345,12 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
                }
 
                /* Lets do the pub key stuff :-) */
-               max=0;
                for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
                        {
                        ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-                       if (ri->cert == NULL)
-                               {
-                               PKCS7err(PKCS7_F_PKCS7_DATAINIT,PKCS7_R_MISSING_CERIPEND_INFO);
+                       if (pkcs7_encode_rinfo(ri, key, keylen) <= 0)
                                goto err;
-                               }
-                       pkey=X509_get_pubkey(ri->cert);
-                       jj=EVP_PKEY_size(pkey);
-                       EVP_PKEY_free(pkey);
-                       if (max < jj) max=jj;
                        }
-               if ((tmp=(unsigned char *)OPENSSL_malloc(max)) == NULL)
-                       {
-                       PKCS7err(PKCS7_F_PKCS7_DATAINIT,ERR_R_MALLOC_FAILURE);
-                       goto err;
-                       }
-               for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
-                       {
-                       ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-                       pkey=X509_get_pubkey(ri->cert);
-                       jj=EVP_PKEY_encrypt(tmp,key,keylen,pkey);
-                       EVP_PKEY_free(pkey);
-                       if (jj <= 0)
-                               {
-                               PKCS7err(PKCS7_F_PKCS7_DATAINIT,ERR_R_EVP_LIB);
-                               OPENSSL_free(tmp);
-                               goto err;
-                               }
-                       M_ASN1_OCTET_STRING_set(ri->enc_key,tmp,jj);
-                       }
-               OPENSSL_free(tmp);
                OPENSSL_cleanse(key, keylen);
 
                if (out == NULL)
@@ -298,12 +386,22 @@ err:
        return(out);
        }
 
+static int pkcs7_cmp_ri(PKCS7_RECIP_INFO *ri, X509 *pcert)
+       {
+       int ret;
+       ret = X509_NAME_cmp(ri->issuer_and_serial->issuer,
+                               pcert->cert_info->issuer);
+       if (ret)
+               return ret;
+       return M_ASN1_INTEGER_cmp(pcert->cert_info->serialNumber,
+                                       ri->issuer_and_serial->serial);
+       }
+
 /* int */
 BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
        {
        int i,j;
        BIO *out=NULL,*btmp=NULL,*etmp=NULL,*bio=NULL;
-       unsigned char *tmp=NULL;
        X509_ALGOR *xa;
        ASN1_OCTET_STRING *data_body=NULL;
        const EVP_MD *evp_md;
@@ -393,7 +491,8 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                int max;
                X509_OBJECT ret;
 #endif
-               int jj;
+               unsigned char *ek = NULL;
+               int eklen;
 
                if ((etmp=BIO_new(BIO_f_cipher())) == NULL)
                        {
@@ -408,57 +507,76 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                 * (if any)
                 */
 
-               for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++) {
-                       ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
-                       if(!X509_NAME_cmp(ri->issuer_and_serial->issuer,
-                                       pcert->cert_info->issuer) &&
-                            !M_ASN1_INTEGER_cmp(pcert->cert_info->serialNumber,
-                                       ri->issuer_and_serial->serial)) break;
-                       ri=NULL;
-               }
-               if (ri == NULL) {
-                       PKCS7err(PKCS7_F_PKCS7_DATADECODE,
-                                PKCS7_R_NO_RECIPIENT_MATCHES_CERTIFICATE);
-                       goto err;
-               }
-
-               jj=EVP_PKEY_size(pkey);
-               tmp=(unsigned char *)OPENSSL_malloc(jj+10);
-               if (tmp == NULL)
+               if (pcert)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_MALLOC_FAILURE);
-                       goto err;
+                       for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
+                               {
+                               ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
+                               if (!pkcs7_cmp_ri(ri, pcert))
+                                       break;
+                               ri=NULL;
+                               }
+                       if (ri == NULL)
+                               {
+                               PKCS7err(PKCS7_F_PKCS7_DATADECODE,
+                                     PKCS7_R_NO_RECIPIENT_MATCHES_CERTIFICATE);
+                               goto err;
+                               }
                        }
 
-               jj=EVP_PKEY_decrypt(tmp, M_ASN1_STRING_data(ri->enc_key),
-                       M_ASN1_STRING_length(ri->enc_key), pkey);
-               if (jj <= 0)
+               /* If we haven't got a certificate try each ri in turn */
+
+               if (pcert == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_EVP_LIB);
-                       goto err;
+                       for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
+                               {
+                               ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
+                               if (pkcs7_decrypt_rinfo(&ek, &eklen,
+                                                       ri, pkey) > 0)
+                                       break;
+                               ERR_clear_error();
+                               ri = NULL;
+                               }
+                       if (ri == NULL)
+                               {
+                               PKCS7err(PKCS7_F_PKCS7_DATADECODE,
+                                     PKCS7_R_NO_RECIPIENT_MATCHES_KEY);
+                               goto err;
+                               }
+                       }
+               else
+                       {
+                       if (pkcs7_decrypt_rinfo(&ek, &eklen, ri, pkey) <= 0)
+                               goto err;
                        }
 
                evp_ctx=NULL;
                BIO_get_cipher_ctx(etmp,&evp_ctx);
-               EVP_CipherInit_ex(evp_ctx,evp_cipher,NULL,NULL,NULL,0);
+               if (EVP_CipherInit_ex(evp_ctx,evp_cipher,NULL,NULL,NULL,0) <= 0)
+                       goto err;
                if (EVP_CIPHER_asn1_to_param(evp_ctx,enc_alg->parameter) < 0)
                        goto err;
 
-               if (jj != EVP_CIPHER_CTX_key_length(evp_ctx)) {
+               if (eklen != EVP_CIPHER_CTX_key_length(evp_ctx)) {
                        /* Some S/MIME clients don't use the same key
                         * and effective key length. The key length is
                         * determined by the size of the decrypted RSA key.
                         */
-                       if(!EVP_CIPHER_CTX_set_key_length(evp_ctx, jj))
+                       if(!EVP_CIPHER_CTX_set_key_length(evp_ctx, eklen))
                                {
                                PKCS7err(PKCS7_F_PKCS7_DATADECODE,
                                        PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH);
                                goto err;
                                }
                } 
-               EVP_CipherInit_ex(evp_ctx,NULL,NULL,tmp,NULL,0);
+               if (EVP_CipherInit_ex(evp_ctx,NULL,NULL,ek,NULL,0) <= 0)
+                       goto err;
 
-               OPENSSL_cleanse(tmp,jj);
+               if (ek)
+                       {
+                       OPENSSL_cleanse(ek,eklen);
+                       OPENSSL_free(ek);
+                       }
 
                if (out == NULL)
                        out=etmp;
@@ -504,8 +622,6 @@ err:
                if (bio != NULL) BIO_free_all(bio);
                out=NULL;
                }
-       if (tmp != NULL)
-               OPENSSL_free(tmp);
        return(out);
        }
 
@@ -516,13 +632,13 @@ static BIO *PKCS7_find_digest(EVP_MD_CTX **pmd, BIO *bio, int nid)
                bio=BIO_find_type(bio,BIO_TYPE_MD);
                if (bio == NULL)
                        {
-                       PKCS7err(PKCS7_F_FIND_DIGEST,PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST);
+                       PKCS7err(PKCS7_F_PKCS7_FIND_DIGEST,PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST);
                        return NULL;    
                        }
                BIO_get_md_ctx(bio,pmd);
                if (*pmd == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_INTERNAL_ERROR);
+                       PKCS7err(PKCS7_F_PKCS7_FIND_DIGEST,ERR_R_INTERNAL_ERROR);
                        return NULL;
                        }       
                if (EVP_MD_CTX_type(*pmd) == nid)
@@ -554,12 +670,20 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
        case NID_pkcs7_signedAndEnveloped:
                /* XXXXXXXXXXXXXXXX */
                si_sk=p7->d.signed_and_enveloped->signer_info;
-               os=M_ASN1_OCTET_STRING_new();
+               if (!(os=M_ASN1_OCTET_STRING_new()))
+                       {
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_MALLOC_FAILURE);
+                       goto err;
+                       }
                p7->d.signed_and_enveloped->enc_data->enc_data=os;
                break;
        case NID_pkcs7_enveloped:
                /* XXXXXXXXXXXXXXXX */
-               os=M_ASN1_OCTET_STRING_new();
+               if (!(os=M_ASN1_OCTET_STRING_new()))
+                       {
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_MALLOC_FAILURE);
+                       goto err;
+                       }
                p7->d.enveloped->enc_data->enc_data=os;
                break;
        case NID_pkcs7_signed:
@@ -588,7 +712,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                {
                if ((buf=BUF_MEM_new()) == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_BIO_LIB);
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_BIO_LIB);
                        goto err;
                        }
                for (i=0; i<sk_PKCS7_SIGNER_INFO_num(si_sk); i++)
@@ -610,7 +734,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                        EVP_MD_CTX_copy_ex(&ctx_tmp,mdc);
                        if (!BUF_MEM_grow_clean(buf,EVP_PKEY_size(si->pkey)))
                                {
-                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_BIO_LIB);
+                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_BIO_LIB);
                                goto err;
                                }
 
@@ -630,7 +754,12 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                                if (!PKCS7_get_signed_attribute(si,
                                                        NID_pkcs9_signingTime))
                                        {
-                                       sign_time=X509_gmtime_adj(NULL,0);
+                                       if (!(sign_time=X509_gmtime_adj(NULL,0)))
+                                               {
+                                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,
+                                                       ERR_R_MALLOC_FAILURE);
+                                               goto err;
+                                               }
                                        PKCS7_add_signed_attribute(si,
                                                NID_pkcs9_signingTime,
                                                V_ASN1_UTCTIME,sign_time);
@@ -639,8 +768,19 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                                /* Add digest */
                                md_tmp=EVP_MD_CTX_md(&ctx_tmp);
                                EVP_DigestFinal_ex(&ctx_tmp,md_data,&md_len);
-                               digest=M_ASN1_OCTET_STRING_new();
-                               M_ASN1_OCTET_STRING_set(digest,md_data,md_len);
+                               if (!(digest=M_ASN1_OCTET_STRING_new()))
+                                       {
+                                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,
+                                               ERR_R_MALLOC_FAILURE);
+                                       goto err;
+                                       }
+                               if (!M_ASN1_OCTET_STRING_set(digest,md_data,
+                                                               md_len))
+                                       {
+                                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,
+                                               ERR_R_MALLOC_FAILURE);
+                                       goto err;
+                                       }
                                PKCS7_add_signed_attribute(si,
                                        NID_pkcs9_messageDigest,
                                        V_ASN1_OCTET_STRING,digest);
@@ -654,25 +794,16 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                                OPENSSL_free(abuf);
                                }
 
-#ifndef OPENSSL_NO_DSA
-                       if (si->pkey->type == EVP_PKEY_DSA)
-                               ctx_tmp.digest=EVP_dss1();
-#endif
-#ifndef OPENSSL_NO_ECDSA
-                       if (si->pkey->type == EVP_PKEY_EC)
-                               ctx_tmp.digest=EVP_ecdsa();
-#endif
-
                        if (!EVP_SignFinal(&ctx_tmp,(unsigned char *)buf->data,
                                (unsigned int *)&buf->length,si->pkey))
                                {
-                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_EVP_LIB);
+                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_EVP_LIB);
                                goto err;
                                }
                        if (!ASN1_STRING_set(si->enc_digest,
                                (unsigned char *)buf->data,buf->length))
                                {
-                               PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_ASN1_LIB);
+                               PKCS7err(PKCS7_F_PKCS7_DATAFINAL,ERR_R_ASN1_LIB);
                                goto err;
                                }
                        }
@@ -693,7 +824,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                btmp=BIO_find_type(bio,BIO_TYPE_MEM);
                if (btmp == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATASIGN,PKCS7_R_UNABLE_TO_FIND_MEM_BIO);
+                       PKCS7err(PKCS7_F_PKCS7_DATAFINAL,PKCS7_R_UNABLE_TO_FIND_MEM_BIO);
                        goto err;
                        }
                BIO_get_mem_ptr(btmp,&buf_mem);
@@ -871,12 +1002,6 @@ for (ii=0; ii<md_len; ii++) printf("%02X",md_dat[ii]); printf(" calc\n");
                ret = -1;
                goto err;
                }
-#ifndef OPENSSL_NO_DSA
-       if(pkey->type == EVP_PKEY_DSA) mdc_tmp.digest=EVP_dss1();
-#endif
-#ifndef OPENSSL_NO_ECDSA
-       if (pkey->type == EVP_PKEY_EC) mdc_tmp.digest=EVP_ecdsa();
-#endif
 
        i=EVP_VerifyFinal(&mdc_tmp,os->data,os->length, pkey);
        EVP_PKEY_free(pkey);