[test/recipes] Split test_fuzz into separate recipes

When using `HARNESS_JOBS` to run the tests in parallel, no matter the
level of parallelism that can be used, the monolithic `test_fuzz` takes
a long time to run, conditioning the duration of the whole build.

This commit splits the single `test_fuzz` recipe into separate recipes
for each fuzzer.
The previous mechanism to select individual fuzz tests using the
`FUZZ_TESTS` environment variable is also dropped (and documentation
updated).

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/13307)

diff --git a/fuzz/README.md b/fuzz/README.md
index deb7a431684752fc8d19eb60ccca242051b48e89..6cc7811ad003eac90743b39453e974c8159b54e2 100644 (file)
--- a/fuzz/README.md
+++ b/fuzz/README.md
@@ -114,15 +114,15 @@ To do all the tests of a specific fuzzer such as asn1 you can run
 
     fuzz/asn1-test fuzz/corpora/asn1
 or
-    make test TESTS=fuzz_test FUZZ_TESTS=asn1
+    make test TESTS=fuzz_test_asn1
 
 To run several fuzz tests you can use for instance:
 
-    make test TESTS=test_fuzz FUZZ_TESTS="cmp cms"
+    make test TESTS='test_fuzz_cmp test_fuzz_cms'
 
 To run all fuzz tests you can use:
 
-    make test TESTS=test_fuzz
+    make test TESTS='test_fuzz_*'
 
 Random numbers
 --------------

diff --git a/test/README.md b/test/README.md
index f4f0574aef797626d00b725a69a22089d426d8d3..43f8471120edc66c5d70de034802194ac2dcdaee 100644 (file)
--- a/test/README.md
+++ b/test/README.md
@@ -98,11 +98,11 @@ it's VMS style wildcards)
 
 Run all tests except for the fuzz tests:
 
-    $ make TESTS=-test_fuzz test
+    $ make TESTS='-test_fuzz*' test
 
 or, if you want to be explicit:
 
-    $ make TESTS='alltests -test_fuzz' test
+    $ make TESTS='alltests -test_fuzz*' test
 
 Run all tests that have a name starting with "test_ssl" but not those
 starting with "test_ssl_":
@@ -123,7 +123,7 @@ Run all tests in test groups 80 to 99 except for tests in group 90:
 
 To run specific fuzz tests you can use for instance:
 
-    $ make test TESTS=test_fuzz FUZZ_TESTS="cmp cms"
+    $ make test TESTS='test_fuzz_cmp test_fuzz_cms'
 
 To stochastically verify that the algorithm that produces uniformly distributed
 random numbers is operating correctly (with a false positive rate of 0.01%):

diff --git a/test/recipes/99-test_fuzz.t b/test/recipes/99-test_fuzz.t
deleted file mode 100644 (file)
index 8bacad4..0000000
--- a/test/recipes/99-test_fuzz.t
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env perl
-# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
-#
-# Licensed under the Apache License 2.0 (the "License").  You may not use
-# this file except in compliance with the License.  You can obtain a copy
-# in the file LICENSE in the source distribution or at
-# https://www.openssl.org/source/license.html
-
-use strict;
-use warnings;
-
-use OpenSSL::Test qw/:DEFAULT srctop_file/;
-use OpenSSL::Test::Utils;
-
-setup("test_fuzz");
-
-my @fuzzers = ();
-@fuzzers = split /\s+/, $ENV{FUZZ_TESTS} if $ENV{FUZZ_TESTS};
-
-if (!@fuzzers) {
-    @fuzzers = (
-        # those commented here as very slow could be moved to separate runs
-        'asn1', # very slow
-        'asn1parse', 'bignum', 'bndiv', 'conf','crl',
-        'client', # very slow
-        'server', # very slow
-        'x509'
-        );
-    push @fuzzers, 'cmp' if !disabled("cmp");
-    push @fuzzers, 'cms' if !disabled("cms");
-    push @fuzzers, 'ct' if !disabled("ct");
-}
-
-plan tests => scalar @fuzzers + 1; # one more due to below require_ok(...)
-
-require_ok(srctop_file('test','recipes','fuzz.pl'));
-
-&fuzz_tests(@fuzzers);

diff --git a/test/recipes/99-test_fuzz_asn1.t b/test/recipes/99-test_fuzz_asn1.t
new file mode 100644 (file)
index 0000000..41fc541
--- /dev/null
+++ b/test/recipes/99-test_fuzz_asn1.t
@@ -0,0 +1,22 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "asn1";
+setup("test_fuzz_${fuzzer}");
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_asn1parse.t b/test/recipes/99-test_fuzz_asn1parse.t
new file mode 100644 (file)
index 0000000..8a008bb
--- /dev/null
+++ b/test/recipes/99-test_fuzz_asn1parse.t
@@ -0,0 +1,22 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "asn1parse";
+setup("test_fuzz_${fuzzer}");
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_bignum.t b/test/recipes/99-test_fuzz_bignum.t
new file mode 100644 (file)
index 0000000..190c37b
--- /dev/null
+++ b/test/recipes/99-test_fuzz_bignum.t
@@ -0,0 +1,22 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "bignum";
+setup("test_fuzz_${fuzzer}");
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_bndiv.t b/test/recipes/99-test_fuzz_bndiv.t
new file mode 100644 (file)
index 0000000..4932840
--- /dev/null
+++ b/test/recipes/99-test_fuzz_bndiv.t
@@ -0,0 +1,22 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "bndiv";
+setup("test_fuzz_${fuzzer}");
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_client.t b/test/recipes/99-test_fuzz_client.t
new file mode 100644 (file)
index 0000000..5d147cf
--- /dev/null
+++ b/test/recipes/99-test_fuzz_client.t
@@ -0,0 +1,22 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "client";
+setup("test_fuzz_${fuzzer}");
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_cmp.t b/test/recipes/99-test_fuzz_cmp.t
new file mode 100644 (file)
index 0000000..0fc1d15
--- /dev/null
+++ b/test/recipes/99-test_fuzz_cmp.t
@@ -0,0 +1,25 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "cmp";
+setup("test_fuzz_${fuzzer}");
+
+plan skip_all => "This test requires $fuzzer support"
+    if disabled($fuzzer);
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_cms.t b/test/recipes/99-test_fuzz_cms.t
new file mode 100644 (file)
index 0000000..9c76f46
--- /dev/null
+++ b/test/recipes/99-test_fuzz_cms.t
@@ -0,0 +1,25 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "cms";
+setup("test_fuzz_${fuzzer}");
+
+plan skip_all => "This test requires $fuzzer support"
+    if disabled($fuzzer);
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_conf.t b/test/recipes/99-test_fuzz_conf.t
new file mode 100644 (file)
index 0000000..c96565f
--- /dev/null
+++ b/test/recipes/99-test_fuzz_conf.t
@@ -0,0 +1,22 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "conf";
+setup("test_fuzz_${fuzzer}");
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_crl.t b/test/recipes/99-test_fuzz_crl.t
new file mode 100644 (file)
index 0000000..1a5281a
--- /dev/null
+++ b/test/recipes/99-test_fuzz_crl.t
@@ -0,0 +1,22 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "crl";
+setup("test_fuzz_${fuzzer}");
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_ct.t b/test/recipes/99-test_fuzz_ct.t
new file mode 100644 (file)
index 0000000..bbfb4ac
--- /dev/null
+++ b/test/recipes/99-test_fuzz_ct.t
@@ -0,0 +1,25 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "ct";
+setup("test_fuzz_${fuzzer}");
+
+plan skip_all => "This test requires $fuzzer support"
+    if disabled($fuzzer);
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_server.t b/test/recipes/99-test_fuzz_server.t
new file mode 100644 (file)
index 0000000..0d0f021
--- /dev/null
+++ b/test/recipes/99-test_fuzz_server.t
@@ -0,0 +1,22 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "server";
+setup("test_fuzz_${fuzzer}");
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/99-test_fuzz_x509.t b/test/recipes/99-test_fuzz_x509.t
new file mode 100644 (file)
index 0000000..9a1e3a1
--- /dev/null
+++ b/test/recipes/99-test_fuzz_x509.t
@@ -0,0 +1,22 @@
+#!/usr/bin/env perl
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test::Utils;
+
+my $fuzzer = "x509";
+setup("test_fuzz_${fuzzer}");
+
+plan tests => 2; # one more due to below require_ok(...)
+
+require_ok(srctop_file('test','recipes','fuzz.pl'));
+
+fuzz_ok($fuzzer);

diff --git a/test/recipes/fuzz.pl b/test/recipes/fuzz.pl
index 795d85c1dfccb457e6417f9b1cd7279bb433d762..3f03eef4f72403f555020abc3edf2061912645a2 100644 (file)
--- a/test/recipes/fuzz.pl
+++ b/test/recipes/fuzz.pl
@@ -9,22 +9,17 @@ use strict;
 use warnings;
 
 use OpenSSL::Glob;
-use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test qw/:DEFAULT srctop_dir/;
 
-sub fuzz_tests {
-    my @fuzzers = @_;
+sub fuzz_ok {
+    die "Only one argument accepted" if scalar @_ != 1;
 
-    foreach my $f (@fuzzers) {
-        subtest "Fuzzing $f" => sub {
-            my @dir = glob(srctop_file('fuzz', 'corpora', "$f"));
+    my $f = $_[0];
+    my $d = srctop_dir('fuzz', 'corpora', $f);
 
-            plan skip_all => "No directory fuzz/corpora/$f" unless @dir;
-            plan tests => scalar @dir; # likely 1
-
-            foreach (@dir) {
-                ok(run(fuzz(["$f-test", $_])));
-            }
-        }
+    SKIP: {
+        skip "No directory $d", 1 unless -d $d;
+        ok(run(fuzz(["$f-test", $d])), "Fuzzing $f");
     }
 }