Lazily initialise the compression buffer

With read pipelining we use multiple SSL3_RECORD structures for reading.
There are SSL_MAX_PIPELINES (32) of them defined (typically not all of these
would be used). Each one has a 16k compression buffer allocated! This
results in a significant amount of memory being consumed which, most of the
time, is not needed.  This change swaps the allocation of the compression
buffer to be lazy so that it is only done immediately before it is actually
used.

Reviewed-by: Tim Hudson <tjh@openssl.org>

diff --git a/ssl/record/rec_layer_s3.c b/ssl/record/rec_layer_s3.c
index 1aceed3aa57931d8d801b2819dd26e20eca40d8f..91b8205ee99167177d06b12756b83d2536824142 100644 (file)
--- a/ssl/record/rec_layer_s3.c
+++ b/ssl/record/rec_layer_s3.c
@@ -223,11 +223,6 @@ void RECORD_LAYER_reset_write_sequence(RECORD_LAYER *rl)
     memset(rl->write_sequence, 0, sizeof(rl->write_sequence));
 }
 
-int RECORD_LAYER_setup_comp_buffer(RECORD_LAYER *rl)
-{
-    return SSL3_RECORD_setup((rl)->rrec, SSL_MAX_PIPELINES);
-}
-
 int ssl3_pending(const SSL *s)
 {
     unsigned int i;

diff --git a/ssl/record/record.h b/ssl/record/record.h
index bf3ffa3cfda41b0d29babc03ded8bfebe1abb11d..000fc856212685be239da193d97705f536ed089e 100644 (file)
--- a/ssl/record/record.h
+++ b/ssl/record/record.h
@@ -325,7 +325,6 @@ int RECORD_LAYER_write_pending(RECORD_LAYER *rl);
 int RECORD_LAYER_set_data(RECORD_LAYER *rl, const unsigned char *buf, int len);
 void RECORD_LAYER_reset_read_sequence(RECORD_LAYER *rl);
 void RECORD_LAYER_reset_write_sequence(RECORD_LAYER *rl);
-int RECORD_LAYER_setup_comp_buffer(RECORD_LAYER *rl);
 int RECORD_LAYER_is_sslv2_record(RECORD_LAYER *rl);
 unsigned int RECORD_LAYER_get_rrec_length(RECORD_LAYER *rl);
 __owur int ssl3_pending(const SSL *s);

diff --git a/ssl/record/record_locl.h b/ssl/record/record_locl.h
index 8239549bebdf0e1b91437011654943face4edb21..e5d2784f262a11e9835c6224fa51fa4622700a54 100644 (file)
--- a/ssl/record/record_locl.h
+++ b/ssl/record/record_locl.h
@@ -193,7 +193,6 @@ int ssl3_release_write_buffer(SSL *s);
 
 void SSL3_RECORD_clear(SSL3_RECORD *r, unsigned int num_recs);
 void SSL3_RECORD_release(SSL3_RECORD *r, unsigned int num_recs);
-int SSL3_RECORD_setup(SSL3_RECORD *r, unsigned int num_recs);
 void SSL3_RECORD_set_seq_num(SSL3_RECORD *r, const unsigned char *seq_num);
 int ssl3_get_record(SSL *s);
 __owur int ssl3_do_compress(SSL *ssl, SSL3_RECORD *wr);

diff --git a/ssl/record/ssl3_record.c b/ssl/record/ssl3_record.c
index 33122626ec2114947fc5b47056cede73d4fdc206..c910e690a2ba45605beeefb57242e50af3039762 100644 (file)
--- a/ssl/record/ssl3_record.c
+++ b/ssl/record/ssl3_record.c
@@ -157,24 +157,6 @@ void SSL3_RECORD_release(SSL3_RECORD *r, unsigned int num_recs)
     }
 }
 
-int SSL3_RECORD_setup(SSL3_RECORD *r, unsigned int num_recs)
-{
-    unsigned int i;
-
-    for (i = 0; i < num_recs; i++) {
-        if (r[i].comp == NULL)
-            r[i].comp = (unsigned char *)
-                OPENSSL_malloc(SSL3_RT_MAX_ENCRYPTED_LENGTH);
-        if (r[i].comp == NULL) {
-            if (i > 0)
-                SSL3_RECORD_release(r, i);
-            return 0;
-        }
-    }
-
-    return 1;
-}
-
 void SSL3_RECORD_set_seq_num(SSL3_RECORD *r, const unsigned char *seq_num)
 {
     memcpy(r->seq_num, seq_num, SEQ_NUM_SIZE);
@@ -626,16 +608,23 @@ int ssl3_do_uncompress(SSL *ssl, SSL3_RECORD *rr)
 #ifndef OPENSSL_NO_COMP
     int i;
 
+    if (rr->comp == NULL) {
+        rr->comp = (unsigned char *)
+            OPENSSL_malloc(SSL3_RT_MAX_ENCRYPTED_LENGTH);
+    }
+    if (rr->comp == NULL)
+        return 0;
+
     i = COMP_expand_block(ssl->expand, rr->comp,
                           SSL3_RT_MAX_PLAIN_LENGTH, rr->data,
                           (int)rr->length);
     if (i < 0)
-        return (0);
+        return 0;
     else
         rr->length = i;
     rr->data = rr->comp;
 #endif
-    return (1);
+    return 1;
 }
 
 int ssl3_do_compress(SSL *ssl, SSL3_RECORD *wr)

diff --git a/ssl/s3_enc.c b/ssl/s3_enc.c
index 1c493e28074f2a361549f119f6f69a45374ebe8e..35ef9487f91d28336e9ca21969c478d4d8f670fb 100644 (file)
--- a/ssl/s3_enc.c
+++ b/ssl/s3_enc.c
@@ -251,8 +251,6 @@ int ssl3_change_cipher_state(SSL *s, int which)
                        SSL_R_COMPRESSION_LIBRARY_ERROR);
                 goto err2;
             }
-            if (!RECORD_LAYER_setup_comp_buffer(&s->rlayer))
-                goto err;
         }
 #endif
         RECORD_LAYER_reset_read_sequence(&s->rlayer);

diff --git a/ssl/t1_enc.c b/ssl/t1_enc.c
index 21eb3283da6682a7ff610c46d3447a475e0cf5e4..f2650b0b7721ebc5b83a37b387e4399540037505 100644 (file)
--- a/ssl/t1_enc.c
+++ b/ssl/t1_enc.c
@@ -260,8 +260,6 @@ int tls1_change_cipher_state(SSL *s, int which)
                        SSL_R_COMPRESSION_LIBRARY_ERROR);
                 goto err2;
             }
-            if (!RECORD_LAYER_setup_comp_buffer(&s->rlayer))
-                goto err;
         }
 #endif
         /*