Fix for v3_scts.c
[openssl.git] / engines / e_aep.c
index 5e66fb1f837ca1181b4aa7f3c0b4675052086ea8..a9372268cd4a21b78504e655390d6d579953fb82 100644 (file)
@@ -57,7 +57,7 @@
 #include <string.h>
 
 #include <openssl/e_os2.h>
-#if !defined(OPENSSL_SYS_MSDOS) || defined(__DJGPP__)
+#if !defined(OPENSSL_SYS_MSDOS) || defined(__DJGPP__) || defined(__MINGW32__)
 #include <sys/types.h>
 #include <unistd.h>
 #else
 typedef int pid_t;
 #endif
 
+#if defined(OPENSSL_SYS_NETWARE) && defined(NETWARE_CLIB)
+#define getpid GetThreadID
+extern int GetThreadID(void);
+#elif defined(_WIN32) && !defined(__WATCOMC__)
+#define getpid _getpid
+#endif
+
 #include <openssl/crypto.h>
 #include <openssl/dso.h>
 #include <openssl/engine.h>
 #include <openssl/buffer.h>
+#ifndef OPENSSL_NO_RSA
 #include <openssl/rsa.h>
+#endif
+#ifndef OPENSSL_NO_DSA
 #include <openssl/dsa.h>
+#endif
+#ifndef OPENSSL_NO_DH
+#include <openssl/dh.h>
+#endif
 
 #ifndef OPENSSL_NO_HW
 #ifndef OPENSSL_NO_HW_AEP
@@ -96,12 +110,14 @@ static AEP_RV aep_close_connection(AEP_CONNECTION_HNDL hConnection);
 static AEP_RV aep_close_all_connections(int use_engine_lock, int *in_use);
 
 /* BIGNUM stuff */
+#ifndef OPENSSL_NO_RSA
 static int aep_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
        const BIGNUM *m, BN_CTX *ctx);
 
 static AEP_RV aep_mod_exp_crt(BIGNUM *r,const  BIGNUM *a, const BIGNUM *p,
        const BIGNUM *q, const BIGNUM *dmp1,const BIGNUM *dmq1,
        const BIGNUM *iqmp, BN_CTX *ctx);
+#endif
 
 /* RSA stuff */
 #ifndef OPENSSL_NO_RSA
@@ -109,8 +125,10 @@ static int aep_rsa_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx);
 #endif
 
 /* This function is aliased to mod_exp (with the mont stuff dropped). */
+#ifndef OPENSSL_NO_RSA
 static int aep_mod_exp_mont(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
        const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx);
+#endif
 
 /* DSA stuff */
 #ifndef OPENSSL_NO_DSA
@@ -477,6 +495,7 @@ static int aep_init(ENGINE *e)
 
        if(aep_dso)
                DSO_free(aep_dso);
+       aep_dso = NULL;
                
        p_AEP_OpenConnection    = NULL;
        p_AEP_ModExp            = NULL;
@@ -618,7 +637,7 @@ static int aep_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
        rv = aep_return_connection(hConnection);
        if (rv != AEP_R_OK)
                {
-               AEPHKerr(AEPHK_F_AEP_RAND,AEPHK_R_RETURN_CONNECTION_FAILED); 
+               AEPHKerr(AEPHK_F_AEP_MOD_EXP,AEPHK_R_RETURN_CONNECTION_FAILED); 
                goto err;
                }
 
@@ -627,6 +646,7 @@ static int aep_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
        return to_return;
        }
        
+#ifndef OPENSSL_NO_RSA
 static AEP_RV aep_mod_exp_crt(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
        const BIGNUM *q, const BIGNUM *dmp1,
        const BIGNUM *dmq1,const BIGNUM *iqmp, BN_CTX *ctx)
@@ -656,13 +676,14 @@ static AEP_RV aep_mod_exp_crt(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
        rv = aep_return_connection(hConnection);
        if (rv != AEP_R_OK)
                {
-               AEPHKerr(AEPHK_F_AEP_RAND,AEPHK_R_RETURN_CONNECTION_FAILED); 
+               AEPHKerr(AEPHK_F_AEP_MOD_EXP_CRT,AEPHK_R_RETURN_CONNECTION_FAILED); 
                goto err;
                }
  
  err:
        return rv;
        }
+#endif
        
 
 #ifdef AEPRAND
@@ -818,12 +839,14 @@ static int aep_mod_exp_dsa(DSA *dsa, BIGNUM *r, BIGNUM *a,
        }
 #endif
 
+#ifndef OPENSSL_NO_RSA
 /* This function is aliased to mod_exp (with the mont stuff dropped). */
 static int aep_mod_exp_mont(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
        const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
        {
        return aep_mod_exp(r, a, p, m, ctx);
        }
+#endif
 
 #ifndef OPENSSL_NO_DH
 /* This function is aliased to mod_exp (with the dh and mont dropped). */
@@ -845,11 +868,7 @@ static AEP_RV aep_get_connection(AEP_CONNECTION_HNDL_PTR phConnection)
 
        CRYPTO_w_lock(CRYPTO_LOCK_ENGINE);
 
-#ifndef NETWARE_CLIB
        curr_pid = getpid();
-#else
-       curr_pid = GetThreadID();
-#endif
 
        /*Check if this is the first time this is being called from the current
          process*/
@@ -1033,13 +1052,10 @@ static AEP_RV GetBigNumSize(AEP_VOID_PTR ArbBigNum, AEP_U32* BigNumSize)
        /*Cast the ArbBigNum pointer to our BIGNUM struct*/
        bn = (BIGNUM*) ArbBigNum;
 
-#ifdef SIXTY_FOUR_BIT_LONG
-       *BigNumSize = bn->top << 3;
-#else
-       /*Size of the bignum in bytes is equal to the bn->top (no of 32 bit
-         words) multiplies by 4*/
-       *BigNumSize = bn->top << 2;
-#endif
+       *BigNumSize = bn->top*BN_BYTES;
+
+       if (BN_BYTES>sizeof(AEP_U32) && (bn->d[bn->top-1]>>BN_BITS4)==0)
+               *BigNumSize -= 4;
 
        return AEP_R_OK;
        }
@@ -1048,31 +1064,42 @@ static AEP_RV MakeAEPBigNum(AEP_VOID_PTR ArbBigNum, AEP_U32 BigNumSize,
        unsigned char* AEP_BigNum)
        {
        BIGNUM* bn;
-
-#ifndef SIXTY_FOUR_BIT_LONG
-       unsigned char* buf;
-       int i;
-#endif
+       const union { long one; char little; } is_endian = {1};
+       AEP_U32 i,j;
 
        /*Cast the ArbBigNum pointer to our BIGNUM struct*/
        bn = (BIGNUM*) ArbBigNum;
 
-#ifdef SIXTY_FOUR_BIT_LONG
-       memcpy(AEP_BigNum, bn->d, BigNumSize);
-#else
        /*Must copy data into a (monotone) least significant byte first format
          performing endian conversion if necessary*/
-       for(i=0;i<bn->top;i++)
+       if (is_endian.little && sizeof(bn->d[0])==BN_BYTES)
+               memcpy(AEP_BigNum, bn->d, BigNumSize);
+       else
                {
-               buf = (unsigned char*)&bn->d[i];
+               BN_ULONG di;
 
-               *((AEP_U32*)AEP_BigNum) = (AEP_U32)
-                       ((unsigned) buf[1] << 8 | buf[0]) |
-                       ((unsigned) buf[3] << 8 | buf[2])  << 16;
+               for (i=0; BigNumSize>=BN_BYTES; i++)
+                       {
+                       di = bn->d[i];
+                       for (j=0; j<BN_BYTES; j++)
+                               {
+                               AEP_BigNum[j] = (unsigned char)di;
+                               di>>=8;
+                               }
+                       AEP_BigNum += BN_BYTES;
+                       BigNumSize -= BN_BYTES;
+                       }
 
-               AEP_BigNum += 4;
+               if (BigNumSize)
+                       {
+                       di = bn->d[i];
+                       for (j=0; j<BigNumSize; j++)
+                               {
+                               AEP_BigNum[j] = (unsigned char)di;
+                               di>>=8;
+                               }
+                       }
                }
-#endif
 
        return AEP_R_OK;
        }
@@ -1082,36 +1109,46 @@ static AEP_RV ConvertAEPBigNum(void* ArbBigNum, AEP_U32 BigNumSize,
        unsigned char* AEP_BigNum)
        {
        BIGNUM* bn;
-#ifndef SIXTY_FOUR_BIT_LONG
-       int i;
-#endif
+       const union { long one; char little; } is_endian = {1};
+       int i,j,top;
 
        bn = (BIGNUM*)ArbBigNum;
 
        /*Expand the result bn so that it can hold our big num.
          Size is in bits*/
-       bn_expand(bn, (int)(BigNumSize << 3));
+       top = (BigNumSize+BN_BYTES-1)/BN_BYTES;
+       bn_expand(bn, top);
+       bn->top = top;
+       bn->d[top-1] = 0;
 
-#ifdef SIXTY_FOUR_BIT_LONG
-       bn->top = BigNumSize >> 3;
-       
-       if((BigNumSize & 7) != 0)
-               bn->top++;
+       if (is_endian.little && sizeof(bn->d[0])==BN_BYTES)
+               memcpy(bn->d, AEP_BigNum, BigNumSize);
+       else
+               {
+               BN_ULONG di;
 
-       memset(bn->d, 0, bn->top << 3); 
+               for (i=0; BigNumSize>=BN_BYTES; i++)
+                       {
+                       for (di=0,j=BN_BYTES; j!=0; )
+                               {
+                               di <<= 8;
+                               di |= AEP_BigNum[--j];
+                               }
+                       bn->d[i] = di;
+                       AEP_BigNum += BN_BYTES;
+                       BigNumSize -= BN_BYTES;
+                       }
 
-       memcpy(bn->d, AEP_BigNum, BigNumSize);
-#else
-       bn->top = BigNumSize >> 2;
-       for(i=0;i<bn->top;i++)
-               {
-               bn->d[i] = (AEP_U32)
-                       ((unsigned) AEP_BigNum[3] << 8 | AEP_BigNum[2]) << 16 |
-                       ((unsigned) AEP_BigNum[1] << 8 | AEP_BigNum[0]);
-               AEP_BigNum += 4;
+               if (BigNumSize)
+                       {
+                       for (di=0,j=BigNumSize; j!=0; )
+                               {
+                               di <<= 8;
+                               di |= AEP_BigNum[--j];
+                               }
+                       bn->d[i] = di;
+                       }
                }
-#endif
 
        return AEP_R_OK;
 }