perf: Avoid taking too long time with high thread counts

We change the tests to perform the same number
of iterations regardless of the thread counts.
Although this raises the running time for small
number of threads, this avoids very long running
times for 100 or more threads.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/tools/pull/162)

diff --git a/perf/README b/perf/README
index d12c5207afd13c192b00bd0da008e016cc146ede..697bbc3b2fa18ce9058c911541fb9592f7c3cec5 100644 (file)
--- a/perf/README
+++ b/perf/README
@@ -41,16 +41,16 @@ labels.
 randbytes
 ---------
 
-The randbytes test repeated calls the RAND_bytes() function in blocks of 100
-calls, and 100 blocks per thread. The number of threads to use is provided as
-an argument and the test reports the average time take to execute a block of 100
-RAND_bytes() calls.
+The randbytes test does 10000 calls of the RAND_bytes() function divided
+evenly among multiple threads. The number of threads to use is provided as
+an argument and the test reports the average time take to execute a block of
+1000 RAND_bytes() calls.
 
 handshake
 ---------
 
-Performs a combined in-memory client and server handshake. Each thread performs
-1000 such handshakes. It takes 2 arguments:
+Performs a combined in-memory client and server handshake. In total 100000
+handshakes are performed divided evenly among each thread. It takes 2 arguments:
 
 certsdir - A directory where 2 files exist (servercert.pem and serverkey.pem) for
 the server certificate and key. The test/certs directory of the main OpenSSL
@@ -58,25 +58,28 @@ source repository contains such files for all supported branches.
 
 threadcount - The number of threads to perform handshakes on in the test
 
-The output is two values: the average time taken for a handshake in us, and the
-average handshakes per second performed over the course of the test.
+The output is two values: the average time taken for a single handshake in us,
+and the average number of simultaneous handshakes per second performed over the
+course of the test.
 
 sslnew
 ------
 
 The sslnew test repeatedly constructs a new SSL object and associates it with a
 newly constructed read BIO and a newly constructed write BIO, and finally frees
-them again. It does this in blocks of 100 sets of calls, and 100 blocks per
-threads. The number of threads to use is provided as an argument and the test
-reports the average time taken to execute a block of 100 construction/free calls.
+them again. It does 100000 repetitions divided evenly among each thread.
+The number of threads to use is provided as an argument and the test
+reports the average time taken to execute a block of 1000 construction/free
+calls.
 
 newrawkey
 ---------
 
 The newrawkey test repeatedly calls the EVP_PKEY_new_raw_public_key_ex()
-function in blocks of 100 calls, and 100 blocks per thread. The number of
-threads to use is provided as an argument and the test reports the average time
-take to execute a block of 100 EVP_PKEY_new_raw_public_key_ex() calls.
+function. It does 100000 repetitions divided evenly among each thread. The
+number of threads to use is provided as an argument and the test reports the
+average time take to execute a block of 1000 EVP_PKEY_new_raw_public_key_ex()
+calls.
 
 Note that this test does not support OpenSSL 1.1.1.
 
@@ -84,9 +87,9 @@ rsasign
 -------
 
 The rsasign test repeatedly calls the EVP_PKEY_sign_init()/EVP_PKEY_sign()
-functions in blocks of 100 calls, and 100 blocks per thread, using a 512 bit RSA
-key. The number of threads to use is provided as an argument and the test
-reports the average time take to execute a block of 100
+functions, using a 512 bit RSA key. It does 100000 repetitions divided evenly
+among each thread. The number of threads to use is provided as an argument and
+the test reports the average time take to execute a block of 1000
 EVP_PKEY_sign_init()/EVP_PKEY_sign() calls.
 
 x509storeissuer
@@ -97,12 +100,21 @@ is used in certificate chain building as part of a verify operation). The test
 assumes that the default certificates directly exists but is empty. For a
 default configuration this is "/usr/local/ssl/certs". The test takes the number
 of threads to use as an argument and the test reports the average time take to
-execute a block of 100 X509_STORE_CTX_get1_issuer() calls.
+execute a block of 1000 X509_STORE_CTX_get1_issuer() calls.
 
 providerdoall
 -------------
 
-The providerdoall test repeatedly calls the OSSL_PROVIDER_do_all() function in
-blocks of 100 calls, and 100 blocks per thread. The number of threads to use is
-provided as an argument and the test reports the average time take to execute a
-block of 100 OSSL_PROVIDER_do_all() calls.
+The providerdoall test repeatedly calls the OSSL_PROVIDER_do_all() function.
+It does 100000 repetitions divided evenly among each thread. The number of
+threads to use is provided as an argument and the test reports the average time
+take to execute a block of 1000 OSSL_PROVIDER_do_all() calls.
+
+pemread
+-------------
+
+The pemread test repeatedly calls the PEM_read_bio_PrivateKey() function on
+a memory BIO with a private RSA key. It does 100000 repetitions divided evenly
+among each thread. The number of threads to use is provided as an argument and
+the test reports the average time take to execute a block of 1000
+PEM_read_bio_PrivateKey() calls.

diff --git a/perf/handshake.c b/perf/handshake.c
index e18f34965dfe955e315b8cf69ba2342c89c10ee2..f1891dbef674946258f4ff95f4d2863870a4f8f9 100644 (file)
--- a/perf/handshake.c
+++ b/perf/handshake.c
@@ -13,7 +13,7 @@
 #include <openssl/ssl.h>
 #include "perflib/perflib.h"
 
-#define NUM_HANDSHAKES_PER_THREAD         1000
+#define NUM_HANDSHAKES_PER_RUN        100000
 
 int err = 0;
 
@@ -21,6 +21,8 @@ static SSL_CTX *sctx = NULL, *cctx = NULL;
 
 OSSL_TIME *times;
 
+static int threadcount;
+
 static void do_handshake(size_t num)
 {
     SSL *clientssl = NULL, *serverssl = NULL;
@@ -30,7 +32,7 @@ static void do_handshake(size_t num)
 
     start = ossl_time_now();
 
-    for (i = 0; i < NUM_HANDSHAKES_PER_THREAD; i++) {
+    for (i = 0; i < NUM_HANDSHAKES_PER_RUN / threadcount; i++) {
         ret = perflib_create_ssl_objects(sctx, cctx, &serverssl, &clientssl,
                                          NULL, NULL);
         ret &= perflib_create_ssl_connection(serverssl, clientssl,
@@ -48,7 +50,6 @@ static void do_handshake(size_t num)
 
 int main(int argc, char *argv[])
 {
-    int threadcount;
     double persec;
     OSSL_TIME duration, av;
     uint64_t us;
@@ -111,9 +112,9 @@ int main(int argc, char *argv[])
     av = times[0];
     for (i = 1; i < threadcount; i++)
         av = ossl_time_add(av, times[i]);
-    av = ossl_time_divide(av, NUM_HANDSHAKES_PER_THREAD * threadcount);
+    av = ossl_time_divide(av, NUM_HANDSHAKES_PER_RUN);
 
-    persec = ((NUM_HANDSHAKES_PER_THREAD * threadcount * OSSL_TIME_SECOND)
+    persec = ((NUM_HANDSHAKES_PER_RUN * OSSL_TIME_SECOND)
              / (double)ossl_time2ticks(duration));
 
     if (terse) {

diff --git a/perf/newrawkey.c b/perf/newrawkey.c
index 10115a3879ca9c404158af10936e34900b5a3bda..42f85b8083775c93ff5c74f5a6d7fd092234c79a 100644 (file)
--- a/perf/newrawkey.c
+++ b/perf/newrawkey.c
@@ -13,9 +13,10 @@
 #include <openssl/evp.h>
 #include "perflib/perflib.h"
 
-#define NUM_CALLS_PER_BLOCK         100
-#define NUM_CALL_BLOCKS_PER_THREAD  100
-#define NUM_CALLS_PER_THREAD        (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_THREAD)
+#define NUM_CALLS_PER_BLOCK         1000
+#define NUM_CALL_BLOCKS_PER_RUN     100
+#define NUM_CALLS_PER_RUN           (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_RUN)
+
 
 int err = 0;
 
@@ -25,12 +26,14 @@ static unsigned char buf[32] = {
     0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f
 };
 
+static int threadcount;
+
 void do_newrawkey(size_t num)
 {
     int i;
     EVP_PKEY *pkey;
 
-    for (i = 0; i < NUM_CALLS_PER_THREAD; i++) {
+    for (i = 0; i < NUM_CALLS_PER_RUN / threadcount; i++) {
         pkey = EVP_PKEY_new_raw_public_key_ex(NULL, "X25519", NULL, buf,
                                               sizeof(buf));
         if (pkey == NULL)
@@ -42,7 +45,6 @@ void do_newrawkey(size_t num)
 
 int main(int argc, char *argv[])
 {
-    int threadcount;
     OSSL_TIME duration;
     uint64_t us;
     double avcalltime;
@@ -80,7 +82,7 @@ int main(int argc, char *argv[])
 
     us = ossl_time2us(duration);
 
-    avcalltime = (double)us / (NUM_CALL_BLOCKS_PER_THREAD * threadcount);
+    avcalltime = (double)us / NUM_CALL_BLOCKS_PER_RUN;
 
     if (terse)
         printf("%lf\n", avcalltime);

diff --git a/perf/pemread.c b/perf/pemread.c
index 73bad947e8bb5b67268cbd789843b42ce1fb7b8e..9a43369fbd4653316644c1b1ed2ec3085904eb09 100644 (file)
--- a/perf/pemread.c
+++ b/perf/pemread.c
@@ -15,9 +15,10 @@
 #include <openssl/crypto.h>
 #include "perflib/perflib.h"
 
-#define NUM_CALLS_PER_BLOCK         100
-#define NUM_CALL_BLOCKS_PER_THREAD  100
-#define NUM_CALLS_PER_THREAD        (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_THREAD)
+#define NUM_CALLS_PER_BLOCK         1000
+#define NUM_CALL_BLOCKS_PER_RUN     100
+#define NUM_CALLS_PER_RUN           (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_RUN)
+
 
 int err = 0;
 
@@ -62,7 +63,7 @@ void do_pemread(size_t num)
      * Technically this includes the EVP_PKEY_free() in the timing - but I
      * think we can live with that
      */
-    for (i = 0; i < NUM_CALLS_PER_THREAD / threadcount; i++) {
+    for (i = 0; i < NUM_CALLS_PER_RUN / threadcount; i++) {
         key = PEM_read_bio_PrivateKey(pem, NULL, NULL, NULL);
         if (key == NULL) {
             printf("Failed to create key: %d\n", i);
@@ -116,7 +117,7 @@ int main(int argc, char *argv[])
 
     us = ossl_time2us(duration);
 
-    avcalltime = (double)us / NUM_CALL_BLOCKS_PER_THREAD;
+    avcalltime = (double)us / NUM_CALL_BLOCKS_PER_RUN;
 
     if (terse)
         printf("%lf\n", avcalltime);

diff --git a/perf/providerdoall.c b/perf/providerdoall.c
index 56390c5077bc6ce61bab3ad5704cd8aa65f0fd4b..e63be24364f2dec4d31ff83a02c29437effc74f5 100644 (file)
--- a/perf/providerdoall.c
+++ b/perf/providerdoall.c
@@ -15,9 +15,9 @@
 #include <openssl/provider.h>
 #include "perflib/perflib.h"
 
-#define NUM_CALLS_PER_BLOCK         100
-#define NUM_CALL_BLOCKS_PER_THREAD  100
-#define NUM_CALLS_PER_THREAD        (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_THREAD)
+#define NUM_CALLS_PER_BLOCK         1000
+#define NUM_CALL_BLOCKS_PER_RUN     100
+#define NUM_CALLS_PER_RUN           (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_RUN)
 
 static int err = 0;
 OSSL_TIME *times;
@@ -30,6 +30,8 @@ static int doit(OSSL_PROVIDER *provider, void *vcount)
     return 1;
 }
 
+static int threadcount;
+
 static void do_providerdoall(size_t num)
 {
     int i;
@@ -39,7 +41,7 @@ static void do_providerdoall(size_t num)
 
     start = ossl_time_now();
 
-    for (i = 0; i < NUM_CALLS_PER_THREAD; i++) {
+    for (i = 0; i < NUM_CALLS_PER_RUN / threadcount; i++) {
         count = 0;
         if (!OSSL_PROVIDER_do_all(NULL, doit, &count) || count != 1) {
             err = 1;
@@ -50,12 +52,12 @@ static void do_providerdoall(size_t num)
     end = ossl_time_now();
 
     times[num] = ossl_time_divide(ossl_time_subtract(end, start),
-                                  NUM_CALL_BLOCKS_PER_THREAD);
+                                  NUM_CALL_BLOCKS_PER_RUN);
 }
 
 int main(int argc, char *argv[])
 {
-    int threadcount, i;
+    int i;
     OSSL_TIME duration, av;
     int terse = 0;
     int argnext;
@@ -99,7 +101,6 @@ int main(int argc, char *argv[])
     av = times[0];
     for (i = 1; i < threadcount; i++)
         av = ossl_time_add(av, times[i]);
-    av = ossl_time_divide(av, threadcount);
 
     if (terse)
         printf("%ld\n", ossl_time2us(av));

diff --git a/perf/randbytes.c b/perf/randbytes.c
index 6357bf00b6741d6dfb49a8dcbad702f2e709d217..c33618dc01d4044c221d48d82490d6f520114da6 100644 (file)
--- a/perf/randbytes.c
+++ b/perf/randbytes.c
@@ -14,25 +14,26 @@
 #include <openssl/crypto.h>
 #include "perflib/perflib.h"
 
-#define NUM_CALLS_PER_BLOCK         100
-#define NUM_CALL_BLOCKS_PER_THREAD  100
-#define NUM_CALLS_PER_THREAD        (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_THREAD)
+#define NUM_CALLS_PER_BLOCK         1000
+#define NUM_CALL_BLOCKS_PER_RUN     100
+#define NUM_CALLS_PER_RUN           (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_RUN)
 
 int err = 0;
 
+static int threadcount;
+
 void do_randbytes(size_t num)
 {
     int i;
     unsigned char buf[32];
 
-    for (i = 0; i < NUM_CALLS_PER_THREAD; i++)
+    for (i = 0; i < NUM_CALLS_PER_RUN / threadcount; i++)
         if (!RAND_bytes(buf, sizeof(buf)))
             err = 1;
 }
 
 int main(int argc, char *argv[])
 {
-    int threadcount;
     OSSL_TIME duration;
     uint64_t us;
     double avcalltime;
@@ -70,7 +71,7 @@ int main(int argc, char *argv[])
 
     us = ossl_time2us(duration);
 
-    avcalltime = (double)us / (NUM_CALL_BLOCKS_PER_THREAD * threadcount);
+    avcalltime = (double)us / NUM_CALL_BLOCKS_PER_RUN;
 
     if (terse)
         printf("%lf\n", avcalltime);

diff --git a/perf/rsasign.c b/perf/rsasign.c
index fdc189af9e376533ed27342e0185be15f1d15ddd..bae4031c30dbe7dc8b45006d29288c7cd12f896b 100644 (file)
--- a/perf/rsasign.c
+++ b/perf/rsasign.c
@@ -16,9 +16,9 @@
 #include <openssl/crypto.h>
 #include "perflib/perflib.h"
 
-#define NUM_CALLS_PER_BLOCK         100
-#define NUM_CALL_BLOCKS_PER_THREAD  100
-#define NUM_CALLS_PER_THREAD        (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_THREAD)
+#define NUM_CALLS_PER_BLOCK         1000
+#define NUM_CALL_BLOCKS_PER_RUN     100
+#define NUM_CALLS_PER_RUN           (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_RUN)
 
 int err = 0;
 EVP_PKEY *rsakey = NULL;
@@ -37,6 +37,8 @@ static const char *rsakeypem =
 
 static const char *tbs = "0123456789abcdefghij"; /* Length of SHA1 digest */
 
+static int threadcount;
+
 void do_rsasign(size_t num)
 {
     int i;
@@ -45,7 +47,7 @@ void do_rsasign(size_t num)
     EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(rsakey, NULL);
     size_t siglen = sizeof(sig);
 
-    for (i = 0; i < NUM_CALLS_PER_THREAD; i++) {
+    for (i = 0; i < NUM_CALLS_PER_RUN; i++) {
         if (EVP_PKEY_sign_init(ctx) <= 0
                 || EVP_PKEY_sign(ctx, sig, &siglen, tbs, SHA_DIGEST_LENGTH) <= 0) {
             err = 1;
@@ -57,7 +59,6 @@ void do_rsasign(size_t num)
 
 int main(int argc, char *argv[])
 {
-    int threadcount;
     OSSL_TIME duration;
     uint64_t us;
     double avcalltime;
@@ -111,7 +112,7 @@ int main(int argc, char *argv[])
 
     us = ossl_time2us(duration);
 
-    avcalltime = (double)us / (NUM_CALL_BLOCKS_PER_THREAD * threadcount);
+    avcalltime = (double)us / NUM_CALL_BLOCKS_PER_RUN;
 
     if (terse)
         printf("%lf\n", avcalltime);

diff --git a/perf/sslnew.c b/perf/sslnew.c
index 014a897c70d093426121d7781e1abc00e8a325a3..7ecbc0b0f00d5e7381f075320f00ed3802195ac1 100644 (file)
--- a/perf/sslnew.c
+++ b/perf/sslnew.c
@@ -14,20 +14,22 @@
 #include <openssl/crypto.h>
 #include "perflib/perflib.h"
 
-#define NUM_CALLS_PER_BLOCK         100
-#define NUM_CALL_BLOCKS_PER_THREAD  100
-#define NUM_CALLS_PER_THREAD        (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_THREAD)
+#define NUM_CALLS_PER_BLOCK         1000
+#define NUM_CALL_BLOCKS_PER_RUN     100
+#define NUM_CALLS_PER_RUN           (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_RUN)
 
 int err = 0;
 static SSL_CTX *ctx;
 
+static int threadcount;
+
 void do_sslnew(size_t num)
 {
     int i;
     SSL *s;
     BIO *rbio, *wbio;
 
-    for (i = 0; i < NUM_CALLS_PER_THREAD; i++) {
+    for (i = 0; i < NUM_CALLS_PER_RUN / threadcount; i++) {
         s = SSL_new(ctx);
         rbio = BIO_new(BIO_s_mem());
         wbio = BIO_new(BIO_s_mem());
@@ -47,7 +49,6 @@ void do_sslnew(size_t num)
 
 int main(int argc, char *argv[])
 {
-    int threadcount;
     OSSL_TIME duration;
     uint64_t us;
     double avcalltime;
@@ -94,7 +95,7 @@ int main(int argc, char *argv[])
 
     us = ossl_time2us(duration);
 
-    avcalltime = (double)us / (NUM_CALL_BLOCKS_PER_THREAD * threadcount);
+    avcalltime = (double)us / NUM_CALL_BLOCKS_PER_RUN;
 
     if (terse)
         printf("%lf\n", avcalltime);

diff --git a/perf/x509storeissuer.c b/perf/x509storeissuer.c
index 7272050f71a388baca0ed1d92f1de481f33c97b5..7115fb6d9eed0f52bf4e7a033a141ce6777ae843 100644 (file)
--- a/perf/x509storeissuer.c
+++ b/perf/x509storeissuer.c
@@ -14,15 +14,17 @@
 #include <openssl/x509.h>
 #include "perflib/perflib.h"
 
-#define NUM_CALLS_PER_BLOCK         100
-#define NUM_CALL_BLOCKS_PER_THREAD  1000
-#define NUM_CALLS_PER_THREAD        (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_THREAD)
+#define NUM_CALLS_PER_BLOCK         1000
+#define NUM_CALL_BLOCKS_PER_RUN     100
+#define NUM_CALLS_PER_RUN           (NUM_CALLS_PER_BLOCK * NUM_CALL_BLOCKS_PER_RUN)
 
 static int err = 0;
 static X509_STORE *store = NULL;
 static X509 *x509 = NULL;
 OSSL_TIME *times;
 
+static int threadcount;
+
 static void do_x509storeissuer(size_t num)
 {
     int i;
@@ -38,7 +40,7 @@ static void do_x509storeissuer(size_t num)
 
     start = ossl_time_now();
 
-    for (i = 0; i < NUM_CALLS_PER_THREAD; i++) {
+    for (i = 0; i < NUM_CALLS_PER_RUN / threadcount; i++) {
         /*
          * We actually expect this to fail. We've not configured any
          * certificates inside our store. We're just testing calling this
@@ -55,7 +57,7 @@ static void do_x509storeissuer(size_t num)
 
     end = ossl_time_now();
     times[num] = ossl_time_divide(ossl_time_subtract(end, start),
-                                  NUM_CALL_BLOCKS_PER_THREAD);
+                                  NUM_CALL_BLOCKS_PER_RUN);
 
  err:
     X509_STORE_CTX_free(ctx);
@@ -136,7 +138,6 @@ int main(int argc, char *argv[])
     av = times[0];
     for (i = 1; i < threadcount; i++)
         av = ossl_time_add(av, times[i]);
-    av = ossl_time_divide(av, threadcount);
 
     if (terse)
         printf("%ld\n", ossl_time2us(av));