2 * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
10 #include "../testutil.h"
17 #include "internal/nelem.h"
18 #include <openssl/bio.h>
20 #include "platform.h" /* From libapps */
23 # define strdup _strdup
28 * Declares the structures needed to register each test case function.
30 typedef struct test_info {
31 const char *test_case_name;
32 int (*test_fn) (void);
33 int (*param_test_fn)(int idx);
40 static TEST_INFO all_tests[1024];
41 static int num_tests = 0;
42 static int show_list = 0;
43 static int single_test = -1;
44 static int single_iter = -1;
48 * A parameterised test runs a loop of test cases.
49 * |num_test_cases| counts the total number of test cases
52 static int num_test_cases = 0;
54 static int process_shared_options(void);
57 void add_test(const char *test_case_name, int (*test_fn) (void))
59 assert(num_tests != OSSL_NELEM(all_tests));
60 all_tests[num_tests].test_case_name = test_case_name;
61 all_tests[num_tests].test_fn = test_fn;
62 all_tests[num_tests].num = -1;
67 void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
70 assert(num_tests != OSSL_NELEM(all_tests));
71 all_tests[num_tests].test_case_name = test_case_name;
72 all_tests[num_tests].param_test_fn = test_fn;
73 all_tests[num_tests].num = num;
74 all_tests[num_tests].subtest = subtest;
76 num_test_cases += num;
79 int subtest_level(void)
84 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
85 static int should_report_leaks(void)
88 * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0
89 * can be used to disable leak checking at runtime.
90 * Note this only works when running the test binary manually;
91 * the test harness always enables OPENSSL_DEBUG_MEMORY.
93 char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY");
95 return mem_debug_env == NULL
96 || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, ""));
100 static int gcd(int a, int b)
110 static void set_seed(int s)
114 seed = (int)time(NULL);
115 test_printf_stdout("%*s# RAND SEED %d\n", subtest_level(), "", seed);
117 test_random_seed(seed);
121 int setup_test_framework(int argc, char *argv[])
123 char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");
124 char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
126 if (TAP_levels != NULL)
127 level = 4 * atoi(TAP_levels);
128 if (test_seed != NULL)
129 set_seed(atoi(test_seed));
131 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
132 if (should_report_leaks()) {
133 CRYPTO_set_mem_debug(1);
134 CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
138 #if defined(OPENSSL_SYS_VMS) && defined(__DECC)
139 argv = copy_argv(&argc, argv);
140 #elif defined(_WIN32)
142 * Replace argv[] with UTF-8 encoded strings.
144 win32_utf8argv(&argc, &argv);
147 if (!opt_init(argc, argv, test_get_options()))
154 * This can only be called after setup() has run, since num_tests and
155 * all_tests[] are setup at this point
157 static int check_single_test_params(char *name, char *testname, char *itname)
161 for (i = 0; i < num_tests; ++i) {
162 if (strcmp(name, all_tests[i].test_case_name) == 0) {
168 single_test = atoi(name);
172 /* if only iteration is specified, assume we want the first test */
173 if (single_test == -1 && single_iter != -1)
176 if (single_test != -1) {
177 if (single_test < 1 || single_test > num_tests) {
178 test_printf_stderr("Invalid -%s value "
179 "(Value must be a valid test name OR a value between %d..%d)\n",
180 testname, 1, num_tests);
184 if (single_iter != -1) {
185 if (all_tests[single_test - 1].num == -1) {
186 test_printf_stderr("-%s option is not valid for test %d:%s\n",
189 all_tests[single_test - 1].test_case_name);
191 } else if (single_iter < 1
192 || single_iter > all_tests[single_test - 1].num) {
193 test_printf_stderr("Invalid -%s value for test %d:%s\t"
194 "(Value must be in the range %d..%d)\n",
196 all_tests[single_test - 1].test_case_name,
197 1, all_tests[single_test - 1].num);
204 static int process_shared_options(void)
206 OPTION_CHOICE_DEFAULT o;
209 char *flag_test = "";
210 char *flag_iter = "";
211 char *testname = NULL;
214 while ((o = opt_next()) != OPT_EOF) {
216 /* Ignore any test options at this level */
222 opt_help(test_get_options());
227 case OPT_TEST_SINGLE:
228 flag_test = opt_flag();
229 testname = opt_arg();
231 case OPT_TEST_ITERATION:
232 flag_iter = opt_flag();
233 if (!opt_int(opt_arg(), &single_iter))
236 case OPT_TEST_INDENT:
237 if (!opt_int(opt_arg(), &value))
242 if (!opt_int(opt_arg(), &value))
248 if (!check_single_test_params(testname, flag_test, flag_iter))
256 int pulldown_test_framework(int ret)
258 set_test_title(NULL);
259 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
260 if (should_report_leaks()
261 && CRYPTO_mem_leaks_cb(openssl_error_cb, NULL) <= 0)
267 static void finalize(int success)
272 ERR_print_errors_cb(openssl_error_cb, NULL);
275 static char *test_title = NULL;
277 void set_test_title(const char *title)
280 test_title = title == NULL ? NULL : strdup(title);
283 PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,
284 const char *description, ...)
291 test_printf_stdout("%*s%s ", level, "", verdict != 0 ? "ok" : "not ok");
292 va_start(ap, description);
293 test_vprintf_stdout(description, ap);
295 if (verdict == TEST_SKIP_CODE)
296 test_printf_stdout(" # skipped");
297 test_printf_stdout("\n");
301 int run_tests(const char *test_prog_name)
305 int ii, i, jj, j, jstep;
306 int permute[OSSL_NELEM(all_tests)];
308 i = process_shared_options();
315 test_printf_stdout("%*s1..0 # Skipped: %s\n", level, "",
317 } else if (show_list == 0 && single_test == -1) {
319 test_printf_stdout("%*s# Subtest: %s\n", level, "", test_prog_name);
320 test_printf_stdout("%*s1..%d\n", level, "", num_tests);
325 for (i = 0; i < num_tests; i++)
328 for (i = num_tests - 1; i >= 1; i--) {
329 j = test_random() % (1 + i);
331 permute[j] = permute[i];
335 for (ii = 0; ii != num_tests; ++ii) {
338 if (single_test != -1 && ((i+1) != single_test)) {
341 else if (show_list) {
342 if (all_tests[i].num != -1) {
343 test_printf_stdout("%d - %s (%d..%d)\n", ii + 1,
344 all_tests[i].test_case_name, 1,
347 test_printf_stdout("%d - %s\n", ii + 1,
348 all_tests[i].test_case_name);
351 } else if (all_tests[i].num == -1) {
352 set_test_title(all_tests[i].test_case_name);
353 verdict = all_tests[i].test_fn();
354 test_verdict(verdict, "%d - %s", ii + 1, test_title);
355 finalize(verdict != 0);
359 int num_failed_inner = 0;
361 verdict = TEST_SKIP_CODE;
363 if (all_tests[i].subtest && single_iter == -1) {
364 test_printf_stdout("%*s# Subtest: %s\n", level, "",
365 all_tests[i].test_case_name);
366 test_printf_stdout("%*s%d..%d\n", level, "", 1,
372 if (seed == 0 || all_tests[i].num < 3)
376 jstep = test_random() % all_tests[i].num;
377 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
379 for (jj = 0; jj < all_tests[i].num; jj++) {
382 j = (j + jstep) % all_tests[i].num;
383 if (single_iter != -1 && ((jj + 1) != single_iter))
385 set_test_title(NULL);
386 v = all_tests[i].param_test_fn(j);
391 } else if (v != TEST_SKIP_CODE && verdict != 0) {
397 if (all_tests[i].subtest) {
398 if (test_title != NULL)
399 test_verdict(v, "%d - %s", jj + 1, test_title);
401 test_verdict(v, "%d - iteration %d", jj + 1, j + 1);
408 test_verdict(verdict, "%d - %s", ii + 1,
409 all_tests[i].test_case_name);
418 * Glue an array of strings together and return it as an allocated string.
419 * Optionally return the whole length of this string in |out_len|
421 char *glue_strings(const char *list[], size_t *out_len)
427 for (i = 0; list[i] != NULL; i++)
428 len += strlen(list[i]);
433 if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1)))
436 for (i = 0; list[i] != NULL; i++)
437 p += strlen(strcpy(p, list[i]));