Fix af_alg engine failure on 32 bit architectures.

[openssl.git] / engines / e_capi.c

diff --git a/engines/e_capi.c b/engines/e_capi.c
index f1a44065b6c0cce22902e7be9ecdfb8bcf6fe8c7..9e9e4f3246be5e6e5b5da9c54f40ea9e809fda16 100644 (file)
--- a/engines/e_capi.c
+++ b/engines/e_capi.c
@@ -7,13 +7,23 @@
  * https://www.openssl.org/source/license.html
  */
 
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
+#ifdef _WIN32
+# ifndef _WIN32_WINNT
+#  define _WIN32_WINNT 0x0400
+# endif
+# include <windows.h>
+# include <wincrypt.h>
+
+# include <stdio.h>
+# include <string.h>
+# include <stdlib.h>
+# include <malloc.h>
+# ifndef alloca
+#  define alloca _alloca
+# endif
 
-#include <openssl/crypto.h>
+# include <openssl/crypto.h>
 
-#ifdef OPENSSL_SYS_WIN32
 # ifndef OPENSSL_NO_CAPIENG
 
 #  include <openssl/buffer.h>
@@ -21,17 +31,6 @@
 #  include <openssl/rsa.h>
 #  include <openssl/dsa.h>
 
-#  ifndef _WIN32_WINNT
-#   define _WIN32_WINNT 0x0400
-#  endif
-
-#  include <windows.h>
-#  include <wincrypt.h>
-#  include <malloc.h>
-#  ifndef alloca
-#   define alloca _alloca
-#  endif
-
 /*
  * This module uses several "new" interfaces, among which is
  * CertGetCertificateContextProperty. CERT_KEY_PROV_INFO_PROP_ID is
@@ -50,7 +49,7 @@
 #   define __COMPILE_CAPIENG
 #  endif                        /* CERT_KEY_PROV_INFO_PROP_ID */
 # endif                         /* OPENSSL_NO_CAPIENG */
-#endif                          /* OPENSSL_SYS_WIN32 */
+#endif                          /* _WIN32 */
 
 #ifdef __COMPILE_CAPIENG
 
@@ -295,6 +294,7 @@ static int capi_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f) (void))
     int ret = 1;
     CAPI_CTX *ctx;
     BIO *out;
+    LPSTR tmpstr;
     if (capi_idx == -1) {
         CAPIerr(CAPI_F_CAPI_CTRL, CAPI_R_ENGINE_NOT_INITIALIZED);
         return 0;
@@ -323,9 +323,15 @@ static int capi_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f) (void))
         break;
 
     case CAPI_CMD_STORE_NAME:
-        OPENSSL_free(ctx->storename);
-        ctx->storename = OPENSSL_strdup(p);
-        CAPI_trace(ctx, "Setting store name to %s\n", p);
+        tmpstr = OPENSSL_strdup(p);
+        if (tmpstr != NULL) {
+            OPENSSL_free(ctx->storename);
+            ctx->storename = tmpstr;
+            CAPI_trace(ctx, "Setting store name to %s\n", p);
+        } else {
+            CAPIerr(CAPI_F_CAPI_CTRL, ERR_R_MALLOC_FAILURE);
+            ret = 0;
+        }
         break;
 
     case CAPI_CMD_STORE_FLAGS:
@@ -345,8 +351,14 @@ static int capi_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f) (void))
         break;
 
     case CAPI_CMD_DEBUG_FILE:
-        ctx->debug_file = OPENSSL_strdup(p);
-        CAPI_trace(ctx, "Setting debug file to %s\n", ctx->debug_file);
+        tmpstr = OPENSSL_strdup(p);
+        if (tmpstr != NULL) {
+            ctx->debug_file = tmpstr;
+            CAPI_trace(ctx, "Setting debug file to %s\n", ctx->debug_file);
+        } else {
+            CAPIerr(CAPI_F_CAPI_CTRL, ERR_R_MALLOC_FAILURE);
+            ret = 0;
+        }
         break;
 
     case CAPI_CMD_KEYTYPE:
@@ -1016,17 +1028,17 @@ static DSA_SIG *capi_dsa_do_sign(const unsigned char *digest, int dlen,
         capi_addlasterror();
         goto err;
     } else {
-        BIGNUM *r = NULL, *s = NULL;
-        ret = DSA_SIG_new();
-        if (ret == NULL)
-            goto err;
-        DSA_SIG_get0(&r, &s, ret);
-        if (!lend_tobn(r, csigbuf, 20)
-            || !lend_tobn(s, csigbuf + 20, 20)) {
-            DSA_SIG_free(ret);
-            ret = NULL;
+        BIGNUM *r = BN_new(), *s = BN_new();
+
+        if (r == NULL || s == NULL
+            || !lend_tobn(r, csigbuf, 20)
+            || !lend_tobn(s, csigbuf + 20, 20)
+            || (ret = DSA_SIG_new()) == NULL) {
+            BN_free(r); /* BN_free checks for BIGNUM * being NULL */
+            BN_free(s);
             goto err;
         }
+        DSA_SIG_set0(ret, r, s);
     }
 
     /* Now cleanup */
@@ -1625,6 +1637,8 @@ static void capi_ctx_free(CAPI_CTX * ctx)
 static int capi_ctx_set_provname(CAPI_CTX * ctx, LPSTR pname, DWORD type,
                                  int check)
 {
+    LPSTR tmpcspname;
+
     CAPI_trace(ctx, "capi_ctx_set_provname, name=%s, type=%d\n", pname, type);
     if (check) {
         HCRYPTPROV hprov;
@@ -1648,8 +1662,13 @@ static int capi_ctx_set_provname(CAPI_CTX * ctx, LPSTR pname, DWORD type,
         }
         CryptReleaseContext(hprov, 0);
     }
+    tmpcspname = OPENSSL_strdup(pname);
+    if (tmpcspname == NULL) {
+        CAPIerr(CAPI_F_CAPI_CTX_SET_PROVNAME, ERR_R_MALLOC_FAILURE);
+        return 0;
+    }
     OPENSSL_free(ctx->cspname);
-    ctx->cspname = OPENSSL_strdup(pname);
+    ctx->cspname = tmpcspname;
     ctx->csptype = type;
     return 1;
 }