trace: don't pretend success if it's not enabled

[openssl.git] / crypto / trace.c

diff --git a/crypto/trace.c b/crypto/trace.c
index 4e93c43defb51139f8ed79b53d6cfe92390c664b..6299a688be54ce5f9306d17a4e0cc9d13ec7171f 100644 (file)
--- a/crypto/trace.c
+++ b/crypto/trace.c
@@ -65,7 +65,7 @@ static int trace_write(BIO *channel,
                        const char *buf, size_t num, size_t *written)
 {
     struct trace_data_st *ctx = BIO_get_data(channel);
-    size_t cnt = ctx->callback(buf, num, ctx->category, OSSL_TRACE_CTRL_DURING,
+    size_t cnt = ctx->callback(buf, num, ctx->category, OSSL_TRACE_CTRL_WRITE,
                                ctx->data);
 
     *written = cnt;
@@ -218,7 +218,7 @@ static int trace_detach_cb(int category, int type, const void *data)
     return 1;
 }
 
-static int set_trace_data(int category, BIO **channel,
+static int set_trace_data(int category, int type, BIO **channel,
                           const char **prefix, const char **suffix,
                           int (*attach_cb)(int, int, const void *),
                           int (*detach_cb)(int, int, const void *))
@@ -253,11 +253,13 @@ static int set_trace_data(int category, BIO **channel,
 
     if (channel != NULL && curr_channel != NULL) {
         BIO_free(curr_channel);
+        trace_channels[category].type = 0;
         trace_channels[category].bio = NULL;
     }
 
     /* Before running callbacks are done, set new data where appropriate */
     if (channel != NULL && *channel != NULL) {
+        trace_channels[category].type = type;
         trace_channels[category].bio = *channel;
     }
 
@@ -313,10 +315,11 @@ void ossl_trace_cleanup(void)
         /* We force the TRACE category to be treated last */
         if (category == OSSL_TRACE_CATEGORY_TRACE)
             continue;
-        set_trace_data(category, &channel, &prefix, &suffix,
+        set_trace_data(category, 0, &channel, &prefix, &suffix,
                        trace_attach_cb, trace_detach_cb);
     }
-    set_trace_data(OSSL_TRACE_CATEGORY_TRACE, &channel, &prefix, &suffix,
+    set_trace_data(OSSL_TRACE_CATEGORY_TRACE, 0, &channel,
+                   &prefix, &suffix,
                    trace_attach_cb, trace_detach_cb);
     CRYPTO_THREAD_lock_free(trace_lock);
 #endif
@@ -325,14 +328,11 @@ void ossl_trace_cleanup(void)
 int OSSL_trace_set_channel(int category, BIO *channel)
 {
 #ifndef OPENSSL_NO_TRACE
-    if (category < 0 || category >= OSSL_TRACE_CATEGORY_NUM
-        || !set_trace_data(category, &channel, NULL, NULL,
-                           trace_attach_cb, trace_detach_cb))
-        return 0;
-
-    trace_channels[category].type = SIMPLE_CHANNEL;
+    if (category >= 0 && category < OSSL_TRACE_CATEGORY_NUM)
+        return set_trace_data(category, SIMPLE_CHANNEL, &channel, NULL, NULL,
+                              trace_attach_cb, trace_detach_cb))
 #endif
-    return 1;
+    return 0;
 }
 
 #ifndef OPENSSL_NO_TRACE
@@ -366,7 +366,7 @@ int OSSL_trace_set_callback(int category, OSSL_trace_cb callback, void *data)
     struct trace_data_st *trace_data = NULL;
 
     if (category < 0 || category >= OSSL_TRACE_CATEGORY_NUM)
-        goto err;
+        return 0;
 
     if (callback != NULL) {
         if ((channel = BIO_new(&trace_method)) == NULL
@@ -381,46 +381,38 @@ int OSSL_trace_set_callback(int category, OSSL_trace_cb callback, void *data)
         BIO_set_data(channel, trace_data);
     }
 
-    if (!set_trace_data(category, &channel, NULL, NULL,
+    if (!set_trace_data(category, CALLBACK_CHANNEL, &channel, NULL, NULL,
                         trace_attach_w_callback_cb, trace_detach_cb))
         goto err;
 
-    trace_channels[category].type = CALLBACK_CHANNEL;
-    goto done;
+    return 1;
 
  err:
     BIO_free(channel);
     OPENSSL_free(trace_data);
-    return 0;
- done:
 #endif
-    return 1;
+
+    return 0;
 }
 
 int OSSL_trace_set_prefix(int category, const char *prefix)
 {
-    int rv = 1;
-
 #ifndef OPENSSL_NO_TRACE
-    if (category >= 0 || category < OSSL_TRACE_CATEGORY_NUM)
-        return set_trace_data(category, NULL, &prefix, NULL,
+    if (category >= 0 && category < OSSL_TRACE_CATEGORY_NUM)
+        return set_trace_data(category, 0, NULL, &prefix, NULL,
                               trace_attach_cb, trace_detach_cb);
-    rv = 0;
 #endif
-    return rv;
+    return 0;
 }
 
 int OSSL_trace_set_suffix(int category, const char *suffix)
 {
-    int rv = 1;
-
 #ifndef OPENSSL_NO_TRACE
-    if (category >= 0 || category < OSSL_TRACE_CATEGORY_NUM)
-        return set_trace_data(category, NULL, NULL, &suffix,
+    if (category >= 0 && category < OSSL_TRACE_CATEGORY_NUM)
+        return set_trace_data(category, 0, NULL, NULL, &suffix,
                               trace_attach_cb, trace_detach_cb);
-    rv = 0;
 #endif
-    return rv;
+    return 0;
 }
 
 #ifndef OPENSSL_NO_TRACE