clear dso pointer in case of an error

[openssl.git] / engines / e_ubsec.c

diff --git a/engines/e_ubsec.c b/engines/e_ubsec.c
index 02927d7b387c733a0d885e2086f4b25be6b6060f..f0e4f736ad5330259541b1195d4209f22b78e18c 100644 (file)
--- a/engines/e_ubsec.c
+++ b/engines/e_ubsec.c
@@ -64,6 +64,10 @@
 #include <openssl/buffer.h>
 #include <openssl/dso.h>
 #include <openssl/engine.h>
 #include <openssl/buffer.h>
 #include <openssl/dso.h>
 #include <openssl/engine.h>

+#include <openssl/rsa.h>
+#include <openssl/dsa.h>
+#include <openssl/dh.h>
+#include <openssl/bn.h>

 
 #ifndef OPENSSL_NO_HW
 #ifndef OPENSSL_NO_HW_UBSEC
 
 #ifndef OPENSSL_NO_HW
 #ifndef OPENSSL_NO_HW_UBSEC


@@ -82,14 +86,14 @@
 static int ubsec_destroy(ENGINE *e);
 static int ubsec_init(ENGINE *e);
 static int ubsec_finish(ENGINE *e);
 static int ubsec_destroy(ENGINE *e);
 static int ubsec_init(ENGINE *e);
 static int ubsec_finish(ENGINE *e);

-static int ubsec_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f)());
+static int ubsec_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f)(void));

 static int ubsec_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
                const BIGNUM *m, BN_CTX *ctx);
 static int ubsec_mod_exp_crt(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
                        const BIGNUM *q, const BIGNUM *dp,
                        const BIGNUM *dq, const BIGNUM *qinv, BN_CTX *ctx);
 #ifndef OPENSSL_NO_RSA
 static int ubsec_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
                const BIGNUM *m, BN_CTX *ctx);
 static int ubsec_mod_exp_crt(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
                        const BIGNUM *q, const BIGNUM *dp,
                        const BIGNUM *dq, const BIGNUM *qinv, BN_CTX *ctx);
 #ifndef OPENSSL_NO_RSA

-static int ubsec_rsa_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa);
+static int ubsec_rsa_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx);

 #endif
 static int ubsec_mod_exp_mont(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
                const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx);
 #endif
 static int ubsec_mod_exp_mont(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
                const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx);


@@ -459,6 +463,7 @@ static int ubsec_init(ENGINE *e)
 err:
        if(ubsec_dso)
                DSO_free(ubsec_dso);
 err:
        if(ubsec_dso)
                DSO_free(ubsec_dso);

+       ubsec_dso = NULL;

        p_UBSEC_ubsec_bytes_to_bits = NULL;
        p_UBSEC_ubsec_bits_to_bytes = NULL;
        p_UBSEC_ubsec_open = NULL;
        p_UBSEC_ubsec_bytes_to_bits = NULL;
        p_UBSEC_ubsec_bits_to_bytes = NULL;
        p_UBSEC_ubsec_open = NULL;


@@ -518,7 +523,7 @@ static int ubsec_finish(ENGINE *e)
        return 1;
        }
 
        return 1;
        }
 

-static int ubsec_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f)())
+static int ubsec_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f)(void))

        {
        int initialised = ((ubsec_dso == NULL) ? 0 : 1);
        switch(cmd)
        {
        int initialised = ((ubsec_dso == NULL) ? 0 : 1);
        switch(cmd)


@@ -566,11 +571,10 @@ static int ubsec_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
                UBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_BN_EXPAND_FAIL);
                return 0;
        }
                UBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_BN_EXPAND_FAIL);
                return 0;
        }

-       memset(r->d, 0, BN_num_bytes(m));

 
        if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0) {
                fd = 0;
 
        if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0) {
                fd = 0;

-               UBSECerr(UBSEC_F_UBSEC_INIT, UBSEC_R_UNIT_FAILURE);
+               UBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_UNIT_FAILURE);

                 return BN_mod_exp(r, a, p, m, ctx);
        }
 
                 return BN_mod_exp(r, a, p, m, ctx);
        }
 


@@ -591,14 +595,10 @@ static int ubsec_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
        }
 
 #ifndef OPENSSL_NO_RSA
        }
 
 #ifndef OPENSSL_NO_RSA

-static int ubsec_rsa_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa)
+static int ubsec_rsa_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx)

        {
        {

-       BN_CTX *ctx;

        int to_return = 0;
 
        int to_return = 0;
 

-       if((ctx = BN_CTX_new()) == NULL)
-               goto err;
-

        if(!rsa->p || !rsa->q || !rsa->dmp1 || !rsa->dmq1 || !rsa->iqmp)
                {
                UBSECerr(UBSEC_F_UBSEC_RSA_MOD_EXP, UBSEC_R_MISSING_KEY_COMPONENTS);
        if(!rsa->p || !rsa->q || !rsa->dmp1 || !rsa->dmq1 || !rsa->iqmp)
                {
                UBSECerr(UBSEC_F_UBSEC_RSA_MOD_EXP, UBSEC_R_MISSING_KEY_COMPONENTS);


@@ -613,11 +613,9 @@ static int ubsec_rsa_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa)
           * Do in software as hardware failed.
           */
           const RSA_METHOD *meth = RSA_PKCS1_SSLeay();
           * Do in software as hardware failed.
           */
           const RSA_METHOD *meth = RSA_PKCS1_SSLeay();

-          to_return = (*meth->rsa_mod_exp)(r0, I, rsa);
+          to_return = (*meth->rsa_mod_exp)(r0, I, rsa, ctx);

        }
 err:
        }
 err:

-       if(ctx)
-               BN_CTX_free(ctx);

        return to_return;
        }
 #endif
        return to_return;
        }
 #endif


@@ -635,18 +633,18 @@ static int ubsec_mod_exp_crt(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
 
        /* Check if hardware can't handle this argument. */
        if (y_len > max_key_len) {
 
        /* Check if hardware can't handle this argument. */
        if (y_len > max_key_len) {

-               UBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_SIZE_TOO_LARGE_OR_TOO_SMALL);
+               UBSECerr(UBSEC_F_UBSEC_MOD_EXP_CRT, UBSEC_R_SIZE_TOO_LARGE_OR_TOO_SMALL);

                return FAIL_TO_SOFTWARE;
        } 
 
        if (!bn_wexpand(r, p->top + q->top + 1)) {
                return FAIL_TO_SOFTWARE;
        } 
 
        if (!bn_wexpand(r, p->top + q->top + 1)) {

-               UBSECerr(UBSEC_F_UBSEC_RSA_MOD_EXP_CRT, UBSEC_R_BN_EXPAND_FAIL);
+               UBSECerr(UBSEC_F_UBSEC_MOD_EXP_CRT, UBSEC_R_BN_EXPAND_FAIL);

                return 0;
        }
 
        if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0) {
                fd = 0;
                return 0;
        }
 
        if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0) {
                fd = 0;

-               UBSECerr(UBSEC_F_UBSEC_INIT, UBSEC_R_UNIT_FAILURE);
+               UBSECerr(UBSEC_F_UBSEC_MOD_EXP_CRT, UBSEC_R_UNIT_FAILURE);

                return FAIL_TO_SOFTWARE;
        }
 
                return FAIL_TO_SOFTWARE;
        }
 


@@ -658,7 +656,7 @@ static int ubsec_mod_exp_crt(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
                (unsigned char *)dq->d, BN_num_bits(dq),
                (unsigned char *)q->d, BN_num_bits(q),
                (unsigned char *)r->d,  &y_len) != 0) {
                (unsigned char *)dq->d, BN_num_bits(dq),
                (unsigned char *)q->d, BN_num_bits(q),
                (unsigned char *)r->d,  &y_len) != 0) {

-               UBSECerr(UBSEC_F_UBSEC_MOD_EXP, UBSEC_R_REQUEST_FAILED);
+               UBSECerr(UBSEC_F_UBSEC_MOD_EXP_CRT, UBSEC_R_REQUEST_FAILED);

                 p_UBSEC_ubsec_close(fd);
                return FAIL_TO_SOFTWARE;
        }
                 p_UBSEC_ubsec_close(fd);
                return FAIL_TO_SOFTWARE;
        }


@@ -752,19 +750,19 @@ static DSA_SIG *ubsec_dsa_do_sign(const unsigned char *dgst, int dlen, DSA *dsa)
 
         if(!bn_wexpand(r, (160+BN_BITS2-1)/BN_BITS2) ||
                   (!bn_wexpand(s, (160+BN_BITS2-1)/BN_BITS2))) {
 
         if(!bn_wexpand(r, (160+BN_BITS2-1)/BN_BITS2) ||
                   (!bn_wexpand(s, (160+BN_BITS2-1)/BN_BITS2))) {

-               UBSECerr(UBSEC_F_UBSEC_DSA_SIGN, UBSEC_R_BN_EXPAND_FAIL);
+               UBSECerr(UBSEC_F_UBSEC_DSA_DO_SIGN, UBSEC_R_BN_EXPAND_FAIL);

                goto err;
        }
 
        if (BN_bin2bn(dgst,dlen,&m) == NULL) {
                goto err;
        }
 
        if (BN_bin2bn(dgst,dlen,&m) == NULL) {

-               UBSECerr(UBSEC_F_UBSEC_DSA_SIGN, UBSEC_R_BN_EXPAND_FAIL);
+               UBSECerr(UBSEC_F_UBSEC_DSA_DO_SIGN, UBSEC_R_BN_EXPAND_FAIL);

                goto err;
        } 
 
        if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0) {
                 const DSA_METHOD *meth;
                fd = 0;
                goto err;
        } 
 
        if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0) {
                 const DSA_METHOD *meth;
                fd = 0;

-               UBSECerr(UBSEC_F_UBSEC_INIT, UBSEC_R_UNIT_FAILURE);
+               UBSECerr(UBSEC_F_UBSEC_DSA_DO_SIGN, UBSEC_R_UNIT_FAILURE);

                 meth = DSA_OpenSSL();
                 to_return =  meth->dsa_do_sign(dgst, dlen, dsa);
                goto err;
                 meth = DSA_OpenSSL();
                 to_return =  meth->dsa_do_sign(dgst, dlen, dsa);
                goto err;


@@ -781,7 +779,7 @@ static DSA_SIG *ubsec_dsa_do_sign(const unsigned char *dgst, int dlen, DSA *dsa)
                (unsigned char *)s->d, &s_len ) != 0) {
                 const DSA_METHOD *meth;
 
                (unsigned char *)s->d, &s_len ) != 0) {
                 const DSA_METHOD *meth;
 

-               UBSECerr(UBSEC_F_UBSEC_DSA_SIGN, UBSEC_R_REQUEST_FAILED);
+               UBSECerr(UBSEC_F_UBSEC_DSA_DO_SIGN, UBSEC_R_REQUEST_FAILED);

                 p_UBSEC_ubsec_close(fd);
                 meth = DSA_OpenSSL();
                 to_return = meth->dsa_do_sign(dgst, dlen, dsa);
                 p_UBSEC_ubsec_close(fd);
                 meth = DSA_OpenSSL();
                 to_return = meth->dsa_do_sign(dgst, dlen, dsa);


@@ -796,7 +794,7 @@ static DSA_SIG *ubsec_dsa_do_sign(const unsigned char *dgst, int dlen, DSA *dsa)
 
        to_return = DSA_SIG_new();
        if(to_return == NULL) {
 
        to_return = DSA_SIG_new();
        if(to_return == NULL) {

-               UBSECerr(UBSEC_F_UBSEC_DSA_SIGN, UBSEC_R_BN_EXPAND_FAIL);
+               UBSECerr(UBSEC_F_UBSEC_DSA_DO_SIGN, UBSEC_R_BN_EXPAND_FAIL);

                goto err;
        }
 
                goto err;
        }
 


@@ -823,7 +821,7 @@ static int ubsec_dsa_verify(const unsigned char *dgst, int dgst_len,
        BN_init(&v);
 
        if(!bn_wexpand(&v, dsa->p->top)) {
        BN_init(&v);
 
        if(!bn_wexpand(&v, dsa->p->top)) {

-               UBSECerr(UBSEC_F_UBSEC_DSA_VERIFY ,UBSEC_R_BN_EXPAND_FAIL);
+               UBSECerr(UBSEC_F_UBSEC_DSA_VERIFY, UBSEC_R_BN_EXPAND_FAIL);

                goto err;
        }
 
                goto err;
        }
 


@@ -834,7 +832,7 @@ static int ubsec_dsa_verify(const unsigned char *dgst, int dgst_len,
        if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0) {
                 const DSA_METHOD *meth;
                fd = 0;
        if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0) {
                 const DSA_METHOD *meth;
                fd = 0;

-               UBSECerr(UBSEC_F_UBSEC_INIT, UBSEC_R_UNIT_FAILURE);
+               UBSECerr(UBSEC_F_UBSEC_DSA_VERIFY, UBSEC_R_UNIT_FAILURE);

                 meth = DSA_OpenSSL();
                 to_return = meth->dsa_do_verify(dgst, dgst_len, sig, dsa);
                goto err;
                 meth = DSA_OpenSSL();
                 to_return = meth->dsa_do_verify(dgst, dgst_len, sig, dsa);
                goto err;


@@ -850,7 +848,7 @@ static int ubsec_dsa_verify(const unsigned char *dgst, int dgst_len,
                (unsigned char *)sig->s->d, BN_num_bits(sig->s),
                (unsigned char *)v.d, &v_len) != 0) {
                 const DSA_METHOD *meth;
                (unsigned char *)sig->s->d, BN_num_bits(sig->s),
                (unsigned char *)v.d, &v_len) != 0) {
                 const DSA_METHOD *meth;

-               UBSECerr(UBSEC_F_UBSEC_DSA_VERIFY , UBSEC_R_REQUEST_FAILED);
+               UBSECerr(UBSEC_F_UBSEC_DSA_VERIFY, UBSEC_R_REQUEST_FAILED);

                 p_UBSEC_ubsec_close(fd);
 
                 meth = DSA_OpenSSL();
                 p_UBSEC_ubsec_close(fd);
 
                 meth = DSA_OpenSSL();


@@ -869,7 +867,7 @@ err:
 #endif
 
 #ifndef OPENSSL_NO_DH
 #endif
 
 #ifndef OPENSSL_NO_DH

-static int ubsec_dh_compute_key (unsigned char *key,const BIGNUM *pub_key,DH *dh)
+static int ubsec_dh_compute_key(unsigned char *key,const BIGNUM *pub_key,DH *dh)

         {
         int      ret      = -1,
                  k_len,
         {
         int      ret      = -1,
                  k_len,


@@ -880,7 +878,7 @@ static int ubsec_dh_compute_key (unsigned char *key,const BIGNUM *pub_key,DH *dh
         if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0)
                 {
                 const DH_METHOD *meth;
         if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0)
                 {
                 const DH_METHOD *meth;

-                ENGINEerr(UBSEC_F_UBSEC_INIT, UBSEC_R_UNIT_FAILURE);
+                UBSECerr(UBSEC_F_UBSEC_DH_COMPUTE_KEY, UBSEC_R_UNIT_FAILURE);

                 meth = DH_OpenSSL();
                 ret = meth->compute_key(key, pub_key, dh);
                 goto err;
                 meth = DH_OpenSSL();
                 ret = meth->compute_key(key, pub_key, dh);
                 goto err;


@@ -894,7 +892,7 @@ static int ubsec_dh_compute_key (unsigned char *key,const BIGNUM *pub_key,DH *dh
                 {
                 /* Hardware's a no go, failover to software */
                 const DH_METHOD *meth;
                 {
                 /* Hardware's a no go, failover to software */
                 const DH_METHOD *meth;

-                ENGINEerr(UBSEC_F_UBSEC_DH_COMPUTE_KEY, UBSEC_R_REQUEST_FAILED);
+                UBSECerr(UBSEC_F_UBSEC_DH_COMPUTE_KEY, UBSEC_R_REQUEST_FAILED);

                 p_UBSEC_ubsec_close(fd);
 
                 meth = DH_OpenSSL();
                 p_UBSEC_ubsec_close(fd);
 
                 meth = DH_OpenSSL();


@@ -910,7 +908,7 @@ err:
         return ret;
         }
 
         return ret;
         }
 

-static int ubsec_dh_generate_key (DH *dh)
+static int ubsec_dh_generate_key(DH *dh)

         {
         int      ret               = 0,
                  random_bits       = 0,
         {
         int      ret               = 0,
                  random_bits       = 0,


@@ -956,7 +954,7 @@ static int ubsec_dh_generate_key (DH *dh)
         if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0)
                 {
                 const DH_METHOD *meth;
         if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0)
                 {
                 const DH_METHOD *meth;

-                ENGINEerr(UBSEC_F_UBSEC_INIT, UBSEC_R_UNIT_FAILURE);
+                UBSECerr(UBSEC_F_UBSEC_DH_GENERATE_KEY, UBSEC_R_UNIT_FAILURE);

                 meth = DH_OpenSSL();
                 ret = meth->generate_key(dh);
                 goto err;
                 meth = DH_OpenSSL();
                 ret = meth->generate_key(dh);
                 goto err;


@@ -972,7 +970,7 @@ static int ubsec_dh_generate_key (DH *dh)
                 /* Hardware's a no go, failover to software */
                 const DH_METHOD *meth;
 
                 /* Hardware's a no go, failover to software */
                 const DH_METHOD *meth;
 

-                ENGINEerr(UBSEC_F_UBSEC_DH_COMPUTE_KEY, UBSEC_R_REQUEST_FAILED);
+                UBSECerr(UBSEC_F_UBSEC_DH_GENERATE_KEY, UBSEC_R_REQUEST_FAILED);

                 p_UBSEC_ubsec_close(fd);
 
                 meth = DH_OpenSSL();
                 p_UBSEC_ubsec_close(fd);
 
                 meth = DH_OpenSSL();


@@ -1004,7 +1002,7 @@ static int ubsec_rand_bytes(unsigned char * buf,
         if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0)
                 {
                 const RAND_METHOD *meth;
         if ((fd = p_UBSEC_ubsec_open(UBSEC_KEY_DEVICE_NAME)) <= 0)
                 {
                 const RAND_METHOD *meth;

-                ENGINEerr(UBSEC_F_UBSEC_INIT, UBSEC_R_UNIT_FAILURE);
+                UBSECerr(UBSEC_F_UBSEC_RAND_BYTES, UBSEC_R_UNIT_FAILURE);

                 num = p_UBSEC_ubsec_bits_to_bytes(num);
                 meth = RAND_SSLeay();
                 meth->seed(buf, num);
                 num = p_UBSEC_ubsec_bits_to_bytes(num);
                 meth = RAND_SSLeay();
                 meth->seed(buf, num);


@@ -1022,7 +1020,7 @@ static int ubsec_rand_bytes(unsigned char * buf,
                 /* Hardware's a no go, failover to software */
                 const RAND_METHOD *meth;
 
                 /* Hardware's a no go, failover to software */
                 const RAND_METHOD *meth;
 

-                ENGINEerr(UBSEC_F_UBSEC_RNG_BYTES, UBSEC_R_REQUEST_FAILED);
+                UBSECerr(UBSEC_F_UBSEC_RAND_BYTES, UBSEC_R_REQUEST_FAILED);

                 p_UBSEC_ubsec_close(fd);
 
                 num = p_UBSEC_ubsec_bits_to_bytes(num);
                 p_UBSEC_ubsec_close(fd);
 
                 num = p_UBSEC_ubsec_bits_to_bytes(num);