make no-des and no-rc2 work.
[openssl.git] / crypto / pkcs7 / pk7_doit.c
index 4222173e36b9360bbc4092c4c55896faad60b963..fa0159ee1d52ab394e29e1f4de2630bd28ce4551 100644 (file)
@@ -61,6 +61,7 @@
 #include <openssl/rand.h>
 #include <openssl/objects.h>
 #include <openssl/x509.h>
+#include <openssl/x509v3.h>
 
 static int add_attribute(STACK_OF(X509_ATTRIBUTE) **sk, int nid, int atrtype,
                         void *value);
@@ -73,7 +74,8 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
        X509_ALGOR *xa;
        const EVP_MD *evp_md;
        const EVP_CIPHER *evp_cipher=NULL;
-       STACK *md_sk=NULL,*rsk=NULL;
+       STACK_OF(X509_ALGOR) *md_sk=NULL;
+       STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL;
        X509_ALGOR *xalg=NULL;
        PKCS7_RECIP_INFO *ri=NULL;
        EVP_PKEY *pkey;
@@ -116,9 +118,9 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 
        if (md_sk != NULL)
                {
-               for (i=0; i<sk_num(md_sk); i++)
+               for (i=0; i<sk_X509_ALGOR_num(md_sk); i++)
                        {
-                       xa=(X509_ALGOR *)sk_value(md_sk,i);
+                       xa=sk_X509_ALGOR_value(md_sk,i);
                        if ((btmp=BIO_new(BIO_f_md())) == NULL)
                                {
                                PKCS7err(PKCS7_F_PKCS7_DATAINIT,ERR_R_BIO_LIB);
@@ -173,9 +175,9 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
 
                /* Lets do the pub key stuff :-) */
                max=0;
-               for (i=0; i<sk_num(rsk); i++)
+               for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
                        {
-                       ri=(PKCS7_RECIP_INFO *)sk_value(rsk,i);
+                       ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
                        if (ri->cert == NULL)
                                {
                                PKCS7err(PKCS7_F_PKCS7_DATAINIT,PKCS7_R_MISSING_CERIPEND_INFO);
@@ -191,9 +193,9 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
                        PKCS7err(PKCS7_F_PKCS7_DATAINIT,ERR_R_MALLOC_FAILURE);
                        goto err;
                        }
-               for (i=0; i<sk_num(rsk); i++)
+               for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++)
                        {
-                       ri=(PKCS7_RECIP_INFO *)sk_value(rsk,i);
+                       ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
                        pkey=X509_get_pubkey(ri->cert);
                        jj=EVP_PKEY_encrypt(tmp,key,keylen,pkey);
                        EVP_PKEY_free(pkey);
@@ -203,7 +205,7 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
                                Free(tmp);
                                goto err;
                                }
-                       ASN1_OCTET_STRING_set(ri->enc_key,tmp,jj);
+                       M_ASN1_OCTET_STRING_set(ri->enc_key,tmp,jj);
                        }
                Free(tmp);
                memset(key, 0, keylen);
@@ -215,30 +217,23 @@ BIO *PKCS7_dataInit(PKCS7 *p7, BIO *bio)
                btmp=NULL;
                }
 
-       if (bio == NULL) /* ??????????? */
-               {
+       if (bio == NULL) {
                if (p7->detached)
                        bio=BIO_new(BIO_s_null());
-               else
-                       {
-                       bio=BIO_new(BIO_s_mem());
-                       /* We need to set this so that when we have read all
-                        * the data, the encrypt BIO, if present, will read
-                        * EOF and encode the last few bytes */
-                       BIO_set_mem_eof_return(bio,0);
-
+               else {
                        if (PKCS7_type_is_signed(p7) &&
-                               PKCS7_type_is_data(p7->d.sign->contents))
-                               {
+                               PKCS7_type_is_data(p7->d.sign->contents)) {
                                ASN1_OCTET_STRING *os;
-
                                os=p7->d.sign->contents->d.data;
-                               if (os->length > 0)
-                                       BIO_write(bio,(char *)os->data,
-                                               os->length);
-                               }
+                               if (os->length > 0) bio = 
+                                       BIO_new_mem_buf(os->data, os->length);
+                       } 
+                       if(bio == NULL) {
+                               bio=BIO_new(BIO_s_mem());
+                               BIO_set_mem_eof_return(bio,0);
                        }
                }
+       }
        BIO_push(out,bio);
        bio=NULL;
        if (0)
@@ -258,16 +253,20 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
        {
        int i,j;
        BIO *out=NULL,*btmp=NULL,*etmp=NULL,*bio=NULL;
-       char *tmp=NULL;
+       unsigned char *tmp=NULL;
        X509_ALGOR *xa;
        ASN1_OCTET_STRING *data_body=NULL;
        const EVP_MD *evp_md;
        const EVP_CIPHER *evp_cipher=NULL;
        EVP_CIPHER_CTX *evp_ctx=NULL;
        X509_ALGOR *enc_alg=NULL;
-       STACK *md_sk=NULL,*rsk=NULL;
+       STACK_OF(X509_ALGOR) *md_sk=NULL;
+       STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL;
        X509_ALGOR *xalg=NULL;
        PKCS7_RECIP_INFO *ri=NULL;
+#ifndef NO_RC2
+       char is_rc2 = 0;
+#endif
 /*     EVP_PKEY *pkey; */
 #if 0
        X509_STORE_CTX s_ctx;
@@ -312,12 +311,22 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                goto err;
                }
 
+       if(EVP_CIPHER_nid(evp_cipher) == NID_rc2_cbc)
+               {
+#ifndef NO_RC2         
+               is_rc2 = 1; 
+#else
+               PKCS7err(PKCS7_F_PKCS7_DATADECODE,PKCS7_R_UNSUPPORTED_CIPHER_TYPE);
+               goto err;
+#endif
+               }
+
        /* We will be checking the signature */
        if (md_sk != NULL)
                {
-               for (i=0; i<sk_num(md_sk); i++)
+               for (i=0; i<sk_X509_ALGOR_num(md_sk); i++)
                        {
-                       xa=(X509_ALGOR *)sk_value(md_sk,i);
+                       xa=sk_X509_ALGOR_value(md_sk,i);
                        if ((btmp=BIO_new(BIO_f_md())) == NULL)
                                {
                                PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_BIO_LIB);
@@ -366,11 +375,11 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                 * (if any)
                 */
 
-               for (i=0; i<sk_num(rsk); i++) {
-                       ri=(PKCS7_RECIP_INFO *)sk_value(rsk,i);
+               for (i=0; i<sk_PKCS7_RECIP_INFO_num(rsk); i++) {
+                       ri=sk_PKCS7_RECIP_INFO_value(rsk,i);
                        if(!X509_NAME_cmp(ri->issuer_and_serial->issuer,
                                        pcert->cert_info->issuer) &&
-                            !ASN1_INTEGER_cmp(pcert->cert_info->serialNumber,
+                            !M_ASN1_INTEGER_cmp(pcert->cert_info->serialNumber,
                                        ri->issuer_and_serial->serial)) break;
                        ri=NULL;
                }
@@ -381,17 +390,15 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                }
 
                jj=EVP_PKEY_size(pkey);
-               tmp=Malloc(jj+10);
+               tmp=(unsigned char *)Malloc(jj+10);
                if (tmp == NULL)
                        {
                        PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_MALLOC_FAILURE);
                        goto err;
                        }
 
-               jj=EVP_PKEY_decrypt((unsigned char *)tmp,
-                       ASN1_STRING_data(ri->enc_key),
-                       ASN1_STRING_length(ri->enc_key),
-                       pkey);
+               jj=EVP_PKEY_decrypt(tmp, M_ASN1_STRING_data(ri->enc_key),
+                       M_ASN1_STRING_length(ri->enc_key), pkey);
                if (jj <= 0)
                        {
                        PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_EVP_LIB);
@@ -404,13 +411,25 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                if (EVP_CIPHER_asn1_to_param(evp_ctx,enc_alg->parameter) < 0)
                        return(NULL);
 
-               if (jj != EVP_CIPHER_CTX_key_length(evp_ctx))
-                       {
-                       PKCS7err(PKCS7_F_PKCS7_DATADECODE,
+               if (jj != EVP_CIPHER_CTX_key_length(evp_ctx)) {
+                       /* HACK: some S/MIME clients don't use the same key
+                        * and effective key length. The key length is
+                        * determined by the size of the decrypted RSA key.
+                        * So we hack things to manually set the RC2 key
+                        * because we currently can't do this with the EVP
+                        * interface.
+                        */
+#ifndef NO_RC2         
+                       if(is_rc2) RC2_set_key(&(evp_ctx->c.rc2_ks),jj, tmp,
+                                       EVP_CIPHER_CTX_key_length(evp_ctx)*8);
+                       else
+#endif
+                               {
+                               PKCS7err(PKCS7_F_PKCS7_DATADECODE,
                                        PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH);
-                       goto err;
-                       }
-               EVP_CipherInit(evp_ctx,NULL,(unsigned char *)tmp,NULL,0);
+                               goto err;
+                               }
+               } else EVP_CipherInit(evp_ctx,NULL,tmp,NULL,0);
 
                memset(tmp,0,jj);
 
@@ -428,6 +447,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
                }
        else 
                {
+#if 0
                bio=BIO_new(BIO_s_mem());
                /* We need to set this so that when we have read all
                 * the data, the encrypt BIO, if present, will read
@@ -436,6 +456,14 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert)
 
                if (data_body->length > 0)
                        BIO_write(bio,(char *)data_body->data,data_body->length);
+#else
+               if (data_body->length > 0)
+                     bio = BIO_new_mem_buf(data_body->data,data_body->length);
+               else {
+                       bio=BIO_new(BIO_s_mem());
+                       BIO_set_mem_eof_return(bio,0);
+               }
+#endif
                }
        BIO_push(out,bio);
        bio=NULL;
@@ -464,7 +492,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
        PKCS7_SIGNER_INFO *si;
        EVP_MD_CTX *mdc,ctx_tmp;
        STACK_OF(X509_ATTRIBUTE) *sk;
-       STACK *si_sk=NULL;
+       STACK_OF(PKCS7_SIGNER_INFO) *si_sk=NULL;
        unsigned char *p,*pp=NULL;
        int x;
        ASN1_OCTET_STRING *os=NULL;
@@ -477,12 +505,12 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
        case NID_pkcs7_signedAndEnveloped:
                /* XXXXXXXXXXXXXXXX */
                si_sk=p7->d.signed_and_enveloped->signer_info;
-               os=ASN1_OCTET_STRING_new();
+               os=M_ASN1_OCTET_STRING_new();
                p7->d.signed_and_enveloped->enc_data->enc_data=os;
                break;
        case NID_pkcs7_enveloped:
                /* XXXXXXXXXXXXXXXX */
-               os=ASN1_OCTET_STRING_new();
+               os=M_ASN1_OCTET_STRING_new();
                p7->d.enveloped->enc_data->enc_data=os;
                break;
        case NID_pkcs7_signed:
@@ -490,7 +518,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                os=p7->d.sign->contents->d.data;
                /* If detached data then the content is excluded */
                if(p7->detached) {
-                       ASN1_OCTET_STRING_free(os);
+                       M_ASN1_OCTET_STRING_free(os);
                        p7->d.sign->contents->d.data = NULL;
                }
                break;
@@ -503,10 +531,9 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                        PKCS7err(PKCS7_F_PKCS7_DATASIGN,ERR_R_BIO_LIB);
                        goto err;
                        }
-               for (i=0; i<sk_num(si_sk); i++)
+               for (i=0; i<sk_PKCS7_SIGNER_INFO_num(si_sk); i++)
                        {
-                       si=(PKCS7_SIGNER_INFO *)
-                               sk_value(si_sk,i);
+                       si=sk_PKCS7_SIGNER_INFO_value(si_sk,i);
                        if (si->pkey == NULL) continue;
 
                        j=OBJ_obj2nid(si->digest_alg->algorithm);
@@ -562,8 +589,8 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                                /* Add digest */
                                md_tmp=EVP_MD_CTX_type(&ctx_tmp);
                                EVP_DigestFinal(&ctx_tmp,md_data,&md_len);
-                               digest=ASN1_OCTET_STRING_new();
-                               ASN1_OCTET_STRING_set(digest,md_data,md_len);
+                               digest=M_ASN1_OCTET_STRING_new();
+                               M_ASN1_OCTET_STRING_set(digest,md_data,md_len);
                                PKCS7_add_signed_attribute(si,
                                        NID_pkcs9_messageDigest,
                                        V_ASN1_OCTET_STRING,digest);
@@ -610,8 +637,17 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
                        goto err;
                        }
                BIO_get_mem_ptr(btmp,&buf_mem);
-               ASN1_OCTET_STRING_set(os,
+               /* Mark the BIO read only then we can use its copy of the data
+                * instead of making an extra copy.
+                */
+               BIO_set_flags(btmp, BIO_FLAGS_MEM_RDONLY);
+               BIO_set_mem_eof_return(btmp, 0);
+               os->data = (unsigned char *)buf_mem->data;
+               os->length = buf_mem->length;
+#if 0
+               M_ASN1_OCTET_STRING_set(os,
                        (unsigned char *)buf_mem->data,buf_mem->length);
+#endif
                }
        if (pp != NULL) Free(pp);
        pp=NULL;
@@ -625,18 +661,10 @@ err:
 int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio,
             PKCS7 *p7, PKCS7_SIGNER_INFO *si)
        {
-/*     PKCS7_SIGNED *s; */
-       ASN1_OCTET_STRING *os;
-       EVP_MD_CTX mdc_tmp,*mdc;
-       unsigned char *pp,*p;
        PKCS7_ISSUER_AND_SERIAL *ias;
        int ret=0,i;
-       int md_type;
-       STACK_OF(X509_ATTRIBUTE) *sk;
        STACK_OF(X509) *cert;
-       BIO *btmp;
        X509 *x509;
-       EVP_PKEY *pkey;
 
        if (PKCS7_type_is_signed(p7))
                {
@@ -665,6 +693,7 @@ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio,
 
        /* Lets verify */
        X509_STORE_CTX_init(ctx,cert_store,x509,cert);
+       X509_STORE_CTX_set_purpose(ctx, X509_PURPOSE_SMIME_SIGN);
        i=X509_verify_cert(ctx);
        if (i <= 0) 
                {
@@ -673,7 +702,30 @@ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio,
                }
        X509_STORE_CTX_cleanup(ctx);
 
-       /* So we like 'x509', lets check the signature. */
+       return PKCS7_signatureVerify(bio, p7, si, x509);
+       err:
+       return ret;
+       }
+
+int PKCS7_signatureVerify(BIO *bio, PKCS7 *p7, PKCS7_SIGNER_INFO *si,
+                                                               X509 *x509)
+       {
+       ASN1_OCTET_STRING *os;
+       EVP_MD_CTX mdc_tmp,*mdc;
+       unsigned char *pp,*p;
+       int ret=0,i;
+       int md_type;
+       STACK_OF(X509_ATTRIBUTE) *sk;
+       BIO *btmp;
+       EVP_PKEY *pkey;
+
+       if (!PKCS7_type_is_signed(p7) && 
+                               !PKCS7_type_is_signedAndEnveloped(p7)) {
+               PKCS7err(PKCS7_F_PKCS7_SIGNATUREVERIFY,
+                                               PKCS7_R_WRONG_PKCS7_TYPE);
+               goto err;
+       }
+
        md_type=OBJ_obj2nid(si->digest_alg->algorithm);
 
        btmp=bio;
@@ -682,13 +734,15 @@ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio,
                if ((btmp == NULL) ||
                        ((btmp=BIO_find_type(btmp,BIO_TYPE_MD)) == NULL))
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATAVERIFY,PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST);
+                       PKCS7err(PKCS7_F_PKCS7_SIGNATUREVERIFY,
+                                       PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST);
                        goto err;
                        }
                BIO_get_md_ctx(btmp,&mdc);
                if (mdc == NULL)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATAVERIFY,PKCS7_R_INTERNAL_ERROR);
+                       PKCS7err(PKCS7_F_PKCS7_SIGNATUREVERIFY,
+                                                       PKCS7_R_INTERNAL_ERROR);
                        goto err;
                        }
                if (EVP_MD_type(EVP_MD_CTX_type(mdc)) == md_type)
@@ -711,7 +765,8 @@ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio,
                message_digest=PKCS7_digest_from_attributes(sk);
                if (!message_digest)
                        {
-                       PKCS7err(PKCS7_F_PKCS7_DATAVERIFY,PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST);
+                       PKCS7err(PKCS7_F_PKCS7_SIGNATUREVERIFY,
+                                       PKCS7_R_UNABLE_TO_FIND_MESSAGE_DIGEST);
                        goto err;
                        }
                if ((message_digest->length != (int)md_len) ||
@@ -725,7 +780,8 @@ for (ii=0; ii<message_digest->length; ii++)
 for (ii=0; ii<md_len; ii++) printf("%02X",md_dat[ii]); printf(" calc\n");
 }
 #endif
-                       PKCS7err(PKCS7_F_PKCS7_DATAVERIFY,PKCS7_R_DIGEST_FAILURE);
+                       PKCS7err(PKCS7_F_PKCS7_SIGNATUREVERIFY,
+                                                       PKCS7_R_DIGEST_FAILURE);
                        ret= -1;
                        goto err;
                        }
@@ -754,7 +810,8 @@ for (ii=0; ii<md_len; ii++) printf("%02X",md_dat[ii]); printf(" calc\n");
        EVP_PKEY_free(pkey);
        if (i <= 0)
                {
-               PKCS7err(PKCS7_F_PKCS7_DATAVERIFY,PKCS7_R_SIGNATURE_FAILURE);
+               PKCS7err(PKCS7_F_PKCS7_SIGNATUREVERIFY,
+                                               PKCS7_R_SIGNATURE_FAILURE);
                ret= -1;
                goto err;
                }
@@ -766,16 +823,16 @@ err:
 
 PKCS7_ISSUER_AND_SERIAL *PKCS7_get_issuer_and_serial(PKCS7 *p7, int idx)
        {
-       STACK *rsk;
+       STACK_OF(PKCS7_RECIP_INFO) *rsk;
        PKCS7_RECIP_INFO *ri;
        int i;
 
        i=OBJ_obj2nid(p7->type);
        if (i != NID_pkcs7_signedAndEnveloped) return(NULL);
        rsk=p7->d.signed_and_enveloped->recipientinfo;
-       ri=(PKCS7_RECIP_INFO *)sk_value(rsk,0);
-       if (sk_num(rsk) <= idx) return(NULL);
-       ri=(PKCS7_RECIP_INFO *)sk_value(rsk,idx);
+       ri=sk_PKCS7_RECIP_INFO_value(rsk,0);
+       if (sk_PKCS7_RECIP_INFO_num(rsk) <= idx) return(NULL);
+       ri=sk_PKCS7_RECIP_INFO_value(rsk,idx);
        return(ri->issuer_and_serial);
        }