From: Richard Levitte Date: Wed, 19 Apr 2017 08:34:54 +0000 (+0200) Subject: TAPify testutil X-Git-Tag: OpenSSL_1_1_1-pre1~1684 X-Git-Url: https://git.openssl.org/?p=openssl.git;a=commitdiff_plain;h=208d721a004026b128dc66300e32e65a9dc7df1d TAPify testutil With the perl test framework comes the output format TAP (Test Anything Protocol, see http://testanything.org/) with extra extension for subtests. This change extends that same output format to any test program using testutil. In this implementation, each test program is seen as a full test that can be used as a subtest. The perl framework passes on the subtest level to the test programs with the environment variable HARNESS_OSSL_LEVEL. Furthermore, and series of tests added with ADD_ALL_TESTS is regarded as another subtest level. Reviewed-by: Rich Salz (Merged from https://github.com/openssl/openssl/pull/3296) --- diff --git a/test/gmdifftest.c b/test/gmdifftest.c index 1d508c0c7c..6869300f3e 100644 --- a/test/gmdifftest.c +++ b/test/gmdifftest.c @@ -60,5 +60,5 @@ void register_tests(void) if (sizeof(time_t) < 8) TEST_info("Skipping; time_t is less than 64-bits"); else - ADD_ALL_TESTS(test_gmtime, 1000000); + ADD_ALL_TESTS_NOSUBTEST(test_gmtime, 1000000); } diff --git a/test/testlib/OpenSSL/Test.pm b/test/testlib/OpenSSL/Test.pm index c4799e8648..f8fcbe906d 100644 --- a/test/testlib/OpenSSL/Test.pm +++ b/test/testlib/OpenSSL/Test.pm @@ -17,7 +17,7 @@ use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); $VERSION = "0.8"; @ISA = qw(Exporter); @EXPORT = (@Test::More::EXPORT, qw(setup run indir cmd app fuzz test - perlapp perltest)); + perlapp perltest subtest)); @EXPORT_OK = (@Test::More::EXPORT_OK, qw(bldtop_dir bldtop_file srctop_dir srctop_file data_file @@ -65,6 +65,7 @@ use File::Spec::Functions qw/file_name_is_absolute curdir canonpath splitdir use File::Path 2.00 qw/rmtree mkpath/; use File::Basename; +my $level = 0; # The name of the test. This is set by setup() and is used in the other # functions to verify that setup() has been used. @@ -454,6 +455,8 @@ sub run { open STDERR, ">", devnull(); } + $ENV{HARNESS_OSSL_LEVEL} = $level + 1; + # The dance we do with $? is the same dance the Unix shells appear to # do. For example, a program that gets aborted (and therefore signals # SIGABRT = 6) will appear to exit with the code 134. We mimic this @@ -1153,4 +1156,13 @@ inspiration from Andy Polyakov Eappro@openssl.org. =cut +no warnings 'redefine'; +sub subtest { + $level++; + + Test::More::subtest @_; + + $level--; +}; + 1; diff --git a/test/testutil.h b/test/testutil.h index 5d96ddd41c..ecf993445d 100644 --- a/test/testutil.h +++ b/test/testutil.h @@ -53,7 +53,12 @@ * Simple parameterized tests. Calls test_function(idx) for each 0 <= idx < num. */ # define ADD_ALL_TESTS(test_function, num) \ - add_all_tests(#test_function, test_function, num) + add_all_tests(#test_function, test_function, num, 1) +/* + * A variant of the same without TAP output. + */ +# define ADD_ALL_TESTS_NOSUBTEST(test_function, num) \ + add_all_tests(#test_function, test_function, num, 0) /*- * Test cases that share common setup should use the helper @@ -131,7 +136,8 @@ void setup_test(void); __owur int finish_test(int ret); void add_test(const char *test_case_name, int (*test_fn) ()); -void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num); +void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num, + int subtest); __owur int run_tests(const char *test_prog_name); /* @@ -369,3 +375,5 @@ int test_flush_stderr(void); extern BIO *bio_out; extern BIO *bio_err; + +int subtest_level(void); diff --git a/test/testutil/driver.c b/test/testutil/driver.c index e70fd21615..4f6b5119c3 100644 --- a/test/testutil/driver.c +++ b/test/testutil/driver.c @@ -23,6 +23,9 @@ typedef struct test_info { int (*test_fn) (); int (*param_test_fn)(int idx); int num; + + /* flags */ + int subtest:1; } TEST_INFO; static TEST_INFO all_tests[1024]; @@ -45,16 +48,24 @@ void add_test(const char *test_case_name, int (*test_fn) ()) } void add_all_tests(const char *test_case_name, int(*test_fn)(int idx), - int num) + int num, int subtest) { assert(num_tests != OSSL_NELEM(all_tests)); all_tests[num_tests].test_case_name = test_case_name; all_tests[num_tests].param_test_fn = test_fn; all_tests[num_tests].num = num; + all_tests[num_tests].subtest = subtest; ++num_tests; num_test_cases += num; } +static int level = 0; + +int subtest_level(void) +{ + return level; +} + #ifndef OPENSSL_NO_CRYPTO_MDEBUG static int should_report_leaks() { @@ -71,7 +82,6 @@ static int should_report_leaks() } #endif - static int err_cb(const char *str, size_t len, void *u) { return test_puts_stderr(str); @@ -79,8 +89,12 @@ static int err_cb(const char *str, size_t len, void *u) void setup_test() { + char *TAP_levels = getenv("HARNESS_OSSL_LEVEL"); + test_open_streams(); + level = TAP_levels != NULL ? 4 * atoi(TAP_levels) : 0; + #ifndef OPENSSL_NO_CRYPTO_MDEBUG if (should_report_leaks()) { CRYPTO_set_mem_debug(1); @@ -121,47 +135,69 @@ static void helper_printf_stdout(const char *fmt, ...) int run_tests(const char *test_prog_name) { int num_failed = 0; + char *verdict = NULL; int i, j; - helper_printf_stdout("%s: %d test case%s\n", test_prog_name, num_test_cases, - num_test_cases == 1 ? "" : "s"); + helper_printf_stdout("%*s%d..%d\n", level, "", 1, num_tests); test_flush_stdout(); for (i = 0; i != num_tests; ++i) { if (all_tests[i].num == -1) { int ret = all_tests[i].test_fn(); + verdict = "ok"; if (!ret) { - helper_printf_stdout("** %s failed **\n--------\n", - all_tests[i].test_case_name); - test_flush_stdout(); + verdict = "not ok"; ++num_failed; } + helper_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1, + all_tests[i].test_case_name); + test_flush_stdout(); finalize(ret); } else { + int num_failed_inner = 0; + + level += 4; + if (all_tests[i].subtest) { + helper_printf_stdout("%*s# Subtest: %s\n", level, "", + all_tests[i].test_case_name); + helper_printf_stdout("%*s%d..%d\n", level, "", 1, + all_tests[i].num); + test_flush_stdout(); + } + for (j = 0; j < all_tests[i].num; j++) { int ret = all_tests[i].param_test_fn(j); - if (!ret) { - helper_printf_stdout("** %s failed test %d\n--------\n", - all_tests[i].test_case_name, j); + if (!ret) + ++num_failed_inner; + + finalize(ret); + + if (all_tests[i].subtest) { + verdict = "ok"; + if (!ret) { + verdict = "not ok"; + ++num_failed_inner; + } + helper_printf_stdout("%*s%s %d\n", level, "", verdict, j + 1); test_flush_stdout(); - ++num_failed; } - finalize(ret); } + + level -= 4; + verdict = "ok"; + if (num_failed_inner) { + verdict = "not ok"; + ++num_failed; + } + helper_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1, + all_tests[i].test_case_name); + test_flush_stdout(); } } - - if (num_failed != 0) { - helper_printf_stdout("%s: %d test%s failed (out of %d)\n", - test_prog_name, num_failed, - num_failed != 1 ? "s" : "", num_test_cases); - test_flush_stdout(); + if (num_failed != 0) return EXIT_FAILURE; - } - helper_printf_stdout(" All tests passed.\n"); - test_flush_stdout(); return EXIT_SUCCESS; } diff --git a/test/testutil/tests.c b/test/testutil/tests.c index f00ec6c19e..67b20a572c 100644 --- a/test/testutil/tests.c +++ b/test/testutil/tests.c @@ -43,6 +43,7 @@ static void test_fail_message(const char *prefix, const char *file, int line, const char *type, const char *fmt, ...) PRINTF_FORMAT(5, 6); +int subtest_level(void); static void helper_printf_stderr(const char *fmt, ...) { @@ -56,6 +57,7 @@ static void helper_printf_stderr(const char *fmt, ...) static void test_fail_message_va(const char *prefix, const char *file, int line, const char *type, const char *fmt, va_list ap) { + helper_printf_stderr("%*s# ", subtest_level(), ""); test_puts_stderr(prefix != NULL ? prefix : "ERROR"); test_puts_stderr(":"); if (type)