Randomise the ordering of the C unit tests.

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3393)

diff --git a/test/bioprinttest.c b/test/bioprinttest.c
index 3034cda6ec9e4f8b3db5741d6510a6ed758d7ab0..2c51b7a8b55d0fd5e8de5863fddc59e637e372a5 100644 (file)
--- a/test/bioprinttest.c
+++ b/test/bioprinttest.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.

  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy


@@ -20,17 +20,20 @@
 
 static int justprint = 0;
 
 
 static int justprint = 0;
 

-static char *fpexpected[][5] = {
-    /*   0 */ { "0.0000e+00", "0.0000", "0", "0.0000E+00", "0" },
-    /*   1 */ { "6.7000e-01", "0.6700", "0.67", "6.7000E-01", "0.67" },
-    /*   2 */ { "6.6667e-01", "0.6667", "0.6667", "6.6667E-01", "0.6667" },
-    /*   3 */ { "6.6667e-04", "0.0007", "0.0006667", "6.6667E-04", "0.0006667" },
-    /*   4 */ { "6.6667e-05", "0.0001", "6.667e-05", "6.6667E-05", "6.667E-05" },
-    /*   5 */ { "6.6667e+00", "6.6667", "6.667", "6.6667E+00", "6.667" },
-    /*   6 */ { "6.6667e+01", "66.6667", "66.67", "6.6667E+01", "66.67" },
-    /*   7 */ { "6.6667e+02", "666.6667", "666.7", "6.6667E+02", "666.7" },
-    /*   8 */ { "6.6667e+03", "6666.6667", "6667", "6.6667E+03", "6667" },
-    /*   9 */ { "6.6667e+04", "66666.6667", "6.667e+04", "6.6667E+04", "6.667E+04" },
+static char *fpexpected[][10][5] = {
+    {
+    /*  00 */ { "0.0000e+00", "0.0000", "0", "0.0000E+00", "0" },
+    /*  01 */ { "6.7000e-01", "0.6700", "0.67", "6.7000E-01", "0.67" },
+    /*  02 */ { "6.6667e-01", "0.6667", "0.6667", "6.6667E-01", "0.6667" },
+    /*  03 */ { "6.6667e-04", "0.0007", "0.0006667", "6.6667E-04", "0.0006667" },
+    /*  04 */ { "6.6667e-05", "0.0001", "6.667e-05", "6.6667E-05", "6.667E-05" },
+    /*  05 */ { "6.6667e+00", "6.6667", "6.667", "6.6667E+00", "6.667" },
+    /*  06 */ { "6.6667e+01", "66.6667", "66.67", "6.6667E+01", "66.67" },
+    /*  07 */ { "6.6667e+02", "666.6667", "666.7", "6.6667E+02", "666.7" },
+    /*  08 */ { "6.6667e+03", "6666.6667", "6667", "6.6667E+03", "6667" },
+    /*  09 */ { "6.6667e+04", "66666.6667", "6.667e+04", "6.6667E+04", "6.667E+04" },
+    },
+    {

     /*  10 */ { "0.00000e+00", "0.00000", "0", "0.00000E+00", "0" },
     /*  11 */ { "6.70000e-01", "0.67000", "0.67", "6.70000E-01", "0.67" },
     /*  12 */ { "6.66667e-01", "0.66667", "0.66667", "6.66667E-01", "0.66667" },
     /*  10 */ { "0.00000e+00", "0.00000", "0", "0.00000E+00", "0" },
     /*  11 */ { "6.70000e-01", "0.67000", "0.67", "6.70000E-01", "0.67" },
     /*  12 */ { "6.66667e-01", "0.66667", "0.66667", "6.66667E-01", "0.66667" },


@@ -41,6 +44,8 @@ static char *fpexpected[][5] = {
     /*  17 */ { "6.66667e+02", "666.66667", "666.67", "6.66667E+02", "666.67" },
     /*  18 */ { "6.66667e+03", "6666.66667", "6666.7", "6.66667E+03", "6666.7" },
     /*  19 */ { "6.66667e+04", "66666.66667", "66667", "6.66667E+04", "66667" },
     /*  17 */ { "6.66667e+02", "666.66667", "666.67", "6.66667E+02", "666.67" },
     /*  18 */ { "6.66667e+03", "6666.66667", "6666.7", "6.66667E+03", "6666.7" },
     /*  19 */ { "6.66667e+04", "66666.66667", "66667", "6.66667E+04", "66667" },

+    },
+    {

     /*  20 */ { "  0.0000e+00", "      0.0000", "           0", "  0.0000E+00", "           0" },
     /*  21 */ { "  6.7000e-01", "      0.6700", "        0.67", "  6.7000E-01", "        0.67" },
     /*  22 */ { "  6.6667e-01", "      0.6667", "      0.6667", "  6.6667E-01", "      0.6667" },
     /*  20 */ { "  0.0000e+00", "      0.0000", "           0", "  0.0000E+00", "           0" },
     /*  21 */ { "  6.7000e-01", "      0.6700", "        0.67", "  6.7000E-01", "        0.67" },
     /*  22 */ { "  6.6667e-01", "      0.6667", "      0.6667", "  6.6667E-01", "      0.6667" },


@@ -51,6 +56,8 @@ static char *fpexpected[][5] = {
     /*  27 */ { "  6.6667e+02", "    666.6667", "       666.7", "  6.6667E+02", "       666.7" },
     /*  28 */ { "  6.6667e+03", "   6666.6667", "        6667", "  6.6667E+03", "        6667" },
     /*  29 */ { "  6.6667e+04", "  66666.6667", "   6.667e+04", "  6.6667E+04", "   6.667E+04" },
     /*  27 */ { "  6.6667e+02", "    666.6667", "       666.7", "  6.6667E+02", "       666.7" },
     /*  28 */ { "  6.6667e+03", "   6666.6667", "        6667", "  6.6667E+03", "        6667" },
     /*  29 */ { "  6.6667e+04", "  66666.6667", "   6.667e+04", "  6.6667E+04", "   6.667E+04" },

+    },
+    {

     /*  30 */ { " 0.00000e+00", "     0.00000", "           0", " 0.00000E+00", "           0" },
     /*  31 */ { " 6.70000e-01", "     0.67000", "        0.67", " 6.70000E-01", "        0.67" },
     /*  32 */ { " 6.66667e-01", "     0.66667", "     0.66667", " 6.66667E-01", "     0.66667" },
     /*  30 */ { " 0.00000e+00", "     0.00000", "           0", " 0.00000E+00", "           0" },
     /*  31 */ { " 6.70000e-01", "     0.67000", "        0.67", " 6.70000E-01", "        0.67" },
     /*  32 */ { " 6.66667e-01", "     0.66667", "     0.66667", " 6.66667E-01", "     0.66667" },


@@ -61,6 +68,8 @@ static char *fpexpected[][5] = {
     /*  37 */ { " 6.66667e+02", "   666.66667", "      666.67", " 6.66667E+02", "      666.67" },
     /*  38 */ { " 6.66667e+03", "  6666.66667", "      6666.7", " 6.66667E+03", "      6666.7" },
     /*  39 */ { " 6.66667e+04", " 66666.66667", "       66667", " 6.66667E+04", "       66667" },
     /*  37 */ { " 6.66667e+02", "   666.66667", "      666.67", " 6.66667E+02", "      666.67" },
     /*  38 */ { " 6.66667e+03", "  6666.66667", "      6666.7", " 6.66667E+03", "      6666.7" },
     /*  39 */ { " 6.66667e+04", " 66666.66667", "       66667", " 6.66667E+04", "       66667" },

+    },
+    {

     /*  40 */ { "0e+00", "0", "0", "0E+00", "0" },
     /*  41 */ { "7e-01", "1", "0.7", "7E-01", "0.7" },
     /*  42 */ { "7e-01", "1", "0.7", "7E-01", "0.7" },
     /*  40 */ { "0e+00", "0", "0", "0E+00", "0" },
     /*  41 */ { "7e-01", "1", "0.7", "7E-01", "0.7" },
     /*  42 */ { "7e-01", "1", "0.7", "7E-01", "0.7" },


@@ -71,6 +80,8 @@ static char *fpexpected[][5] = {
     /*  47 */ { "7e+02", "667", "7e+02", "7E+02", "7E+02" },
     /*  48 */ { "7e+03", "6667", "7e+03", "7E+03", "7E+03" },
     /*  49 */ { "7e+04", "66667", "7e+04", "7E+04", "7E+04" },
     /*  47 */ { "7e+02", "667", "7e+02", "7E+02", "7E+02" },
     /*  48 */ { "7e+03", "6667", "7e+03", "7E+03", "7E+03" },
     /*  49 */ { "7e+04", "66667", "7e+04", "7E+04", "7E+04" },

+    },
+    {

     /*  50 */ { "0.000000e+00", "0.000000", "0", "0.000000E+00", "0" },
     /*  51 */ { "6.700000e-01", "0.670000", "0.67", "6.700000E-01", "0.67" },
     /*  52 */ { "6.666667e-01", "0.666667", "0.666667", "6.666667E-01", "0.666667" },
     /*  50 */ { "0.000000e+00", "0.000000", "0", "0.000000E+00", "0" },
     /*  51 */ { "6.700000e-01", "0.670000", "0.67", "6.700000E-01", "0.67" },
     /*  52 */ { "6.666667e-01", "0.666667", "0.666667", "6.666667E-01", "0.666667" },


@@ -81,6 +92,8 @@ static char *fpexpected[][5] = {
     /*  57 */ { "6.666667e+02", "666.666667", "666.667", "6.666667E+02", "666.667" },
     /*  58 */ { "6.666667e+03", "6666.666667", "6666.67", "6.666667E+03", "6666.67" },
     /*  59 */ { "6.666667e+04", "66666.666667", "66666.7", "6.666667E+04", "66666.7" },
     /*  57 */ { "6.666667e+02", "666.666667", "666.667", "6.666667E+02", "666.667" },
     /*  58 */ { "6.666667e+03", "6666.666667", "6666.67", "6.666667E+03", "6666.67" },
     /*  59 */ { "6.666667e+04", "66666.666667", "66666.7", "6.666667E+04", "66666.7" },

+    },
+    {

     /*  60 */ { "0.0000e+00", "000.0000", "00000000", "0.0000E+00", "00000000" },
     /*  61 */ { "6.7000e-01", "000.6700", "00000.67", "6.7000E-01", "00000.67" },
     /*  62 */ { "6.6667e-01", "000.6667", "000.6667", "6.6667E-01", "000.6667" },
     /*  60 */ { "0.0000e+00", "000.0000", "00000000", "0.0000E+00", "00000000" },
     /*  61 */ { "6.7000e-01", "000.6700", "00000.67", "6.7000E-01", "00000.67" },
     /*  62 */ { "6.6667e-01", "000.6667", "000.6667", "6.6667E-01", "000.6667" },


@@ -91,6 +104,7 @@ static char *fpexpected[][5] = {
     /*  67 */ { "6.6667e+02", "666.6667", "000666.7", "6.6667E+02", "000666.7" },
     /*  68 */ { "6.6667e+03", "6666.6667", "00006667", "6.6667E+03", "00006667" },
     /*  69 */ { "6.6667e+04", "66666.6667", "6.667e+04", "6.6667E+04", "6.667E+04" },
     /*  67 */ { "6.6667e+02", "666.6667", "000666.7", "6.6667E+02", "000666.7" },
     /*  68 */ { "6.6667e+03", "6666.6667", "00006667", "6.6667E+03", "00006667" },
     /*  69 */ { "6.6667e+04", "66666.6667", "6.667e+04", "6.6667E+04", "6.667E+04" },

+    },

 };
 
 typedef struct z_data_st {
 };
 
 typedef struct z_data_st {


@@ -170,7 +184,7 @@ static pw pw_params[] = {
     { 4, "08" }
 };
 
     { 4, "08" }
 };
 

-static int dofptest(int test, double val, const char *width, int prec)
+static int dofptest(int test, int sub, double val, const char *width, int prec)

 {
     static const char *fspecs[] = {
         "e", "f", "g", "E", "G"
 {
     static const char *fspecs[] = {
         "e", "f", "g", "E", "G"


@@ -190,12 +204,12 @@ static int dofptest(int test, double val, const char *width, int prec)
 
         if (justprint) {
             if (i == 0)
 
         if (justprint) {
             if (i == 0)

-                printf("    /* %d */  { \"%s\"", test, result);
+                printf("    /*  %d%d */ { \"%s\"", test, sub, result);

             else
                 printf(", \"%s\"", result);
             else
                 printf(", \"%s\"", result);

-        } else if (!TEST_str_eq(fpexpected[test][i], result)) {
+        } else if (!TEST_str_eq(fpexpected[test][sub][i], result)) {

             TEST_info("test %d format=|%s| exp=|%s|, ret=|%s|",
             TEST_info("test %d format=|%s| exp=|%s|, ret=|%s|",

-                    test, format, fpexpected[test][i], result);
+                    test, format, fpexpected[test][sub][i], result);

             ret = 0;
         }
     }
             ret = 0;
         }
     }


@@ -206,22 +220,25 @@ static int dofptest(int test, double val, const char *width, int prec)
 
 static int test_fp(int i)
 {
 
 static int test_fp(int i)
 {

-    static int t = 0;
+    int t = 0, r;

     const double frac = 2.0 / 3.0;
     const pw *pwp = &pw_params[i];
 
     const double frac = 2.0 / 3.0;
     const pw *pwp = &pw_params[i];
 

-        if (!TEST_true(dofptest(t++, 0.0, pwp->w, pwp->p))
-                || !TEST_true(dofptest(t++, 0.67, pwp->w, pwp->p))
-                || !TEST_true(dofptest(t++, frac, pwp->w, pwp->p))
-                || !TEST_true(dofptest(t++, frac / 1000, pwp->w, pwp->p))
-                || !TEST_true(dofptest(t++, frac / 10000, pwp->w, pwp->p))
-                || !TEST_true(dofptest(t++, 6.0 + frac, pwp->w, pwp->p))
-                || !TEST_true(dofptest(t++, 66.0 + frac, pwp->w, pwp->p))
-                || !TEST_true(dofptest(t++, 666.0 + frac, pwp->w, pwp->p))
-                || !TEST_true(dofptest(t++, 6666.0 + frac, pwp->w, pwp->p))
-                || !TEST_true(dofptest(t++, 66666.0 + frac, pwp->w, pwp->p)))
-            return 0;
-    return 1;
+    if (justprint)
+        printf("    {\n");
+    r = TEST_true(dofptest(i, t++, 0.0, pwp->w, pwp->p))
+        && TEST_true(dofptest(i, t++, 0.67, pwp->w, pwp->p))
+        && TEST_true(dofptest(i, t++, frac, pwp->w, pwp->p))
+        && TEST_true(dofptest(i, t++, frac / 1000, pwp->w, pwp->p))
+        && TEST_true(dofptest(i, t++, frac / 10000, pwp->w, pwp->p))
+        && TEST_true(dofptest(i, t++, 6.0 + frac, pwp->w, pwp->p))
+        && TEST_true(dofptest(i, t++, 66.0 + frac, pwp->w, pwp->p))
+        && TEST_true(dofptest(i, t++, 666.0 + frac, pwp->w, pwp->p))
+        && TEST_true(dofptest(i, t++, 6666.0 + frac, pwp->w, pwp->p))
+        && TEST_true(dofptest(i, t++, 66666.0 + frac, pwp->w, pwp->p));
+    if (justprint)
+        printf("    },\n");
+    return r;

 }
 
 static int test_big(void)
 }
 
 static int test_big(void)

diff --git a/test/testutil/driver.c b/test/testutil/driver.c
index 8587304a7f22ba478608471587316be7ced28ce1..3b62e59554a5d8902d4ddabc864bfaf3ddc9b1ea 100644 (file)
--- a/test/testutil/driver.c
+++ b/test/testutil/driver.c
@@ -32,6 +32,7 @@ typedef struct test_info {
 
 static TEST_INFO all_tests[1024];
 static int num_tests = 0;
 
 static TEST_INFO all_tests[1024];
 static int num_tests = 0;

+static int seed = 0;

 /*
  * A parameterised tests runs a loop of test cases.
  * |num_test_cases| counts the total number of test cases
 /*
  * A parameterised tests runs a loop of test cases.
  * |num_test_cases| counts the total number of test cases


@@ -84,14 +85,34 @@ static int should_report_leaks()
 }
 #endif
 
 }
 #endif
 

+static int gcd(int a, int b)
+{
+    while (b != 0) {
+        int t = b;
+        b = a % b;
+        a = t;
+    }
+    return a;
+}
+

 void setup_test()
 {
     char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
 void setup_test()
 {
     char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");

+    char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");

 
     test_open_streams();
 
     level = TAP_levels != NULL ? 4 * atoi(TAP_levels) : 0;
 
 
     test_open_streams();
 
     level = TAP_levels != NULL ? 4 * atoi(TAP_levels) : 0;
 

+    if (test_seed != NULL) {
+        seed = atoi(test_seed);
+        if (seed <= 0)
+            seed = time(NULL);
+        test_printf_stdout("%*s# RAND SEED %d\n", subtest_level(), "", seed);
+        test_flush_stdout();
+        srand(seed);
+    }
+

 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
     if (should_report_leaks()) {
         CRYPTO_set_mem_debug(1);
 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
     if (should_report_leaks()) {
         CRYPTO_set_mem_debug(1);


@@ -133,7 +154,8 @@ int run_tests(const char *test_prog_name)
 {
     int num_failed = 0;
     char *verdict = NULL;
 {
     int num_failed = 0;
     char *verdict = NULL;

-    int i, j;
+    int ii, i, jj, j, jstep;
+    int permute[OSSL_NELEM(all_tests)];

 
     if (num_tests < 1) {
         test_printf_stdout("%*s1..0 # Skipped: %s\n", level, "",
 
     if (num_tests < 1) {
         test_printf_stdout("%*s1..0 # Skipped: %s\n", level, "",


@@ -145,7 +167,18 @@ int run_tests(const char *test_prog_name)
     }
     test_flush_stdout();
 
     }
     test_flush_stdout();
 

-    for (i = 0; i != num_tests; ++i) {
+    for (i = 0; i < num_tests; i++)
+        permute[i] = i;
+    if (seed != 0)
+        for (i = num_tests - 1; i >= 1; i--) {
+            j = rand() % (1 + i);
+            ii = permute[j];
+            permute[j] = permute[i];
+            permute[i] = ii;
+        }
+
+    for (ii = 0; ii != num_tests; ++ii) {
+        i = permute[ii];

         if (all_tests[i].num == -1) {
             int ret = 0;
 
         if (all_tests[i].num == -1) {
             int ret = 0;
 


@@ -160,7 +193,7 @@ int run_tests(const char *test_prog_name)
                 verdict = "not ok";
                 ++num_failed;
             }
                 verdict = "not ok";
                 ++num_failed;
             }

-            test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1,
+            test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, ii + 1,

                                test_title);
             test_flush_stdout();
             finalize(ret);
                                test_title);
             test_flush_stdout();
             finalize(ret);


@@ -176,9 +209,18 @@ int run_tests(const char *test_prog_name)
                 test_flush_stdout();
             }
 
                 test_flush_stdout();
             }
 

-            for (j = 0; j < all_tests[i].num; j++) {
-                int ret = 0;
+            j = -1;
+            if (seed == 0 || all_tests[i].num < 3)
+                jstep = 1;
+            else
+                do
+                    jstep = rand() % all_tests[i].num;
+                while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
+
+            for (jj = 0; jj < all_tests[i].num; jj++) {
+                int ret;

 
 

+                j = (j + jstep) % all_tests[i].num;

                 set_test_title(NULL);
                 ret = all_tests[i].param_test_fn(j);
 
                 set_test_title(NULL);
                 ret = all_tests[i].param_test_fn(j);
 


@@ -197,11 +239,11 @@ int run_tests(const char *test_prog_name)
                         ++num_failed_inner;
                     }
                     if (test_title != NULL)
                         ++num_failed_inner;
                     }
                     if (test_title != NULL)

-                        test_printf_stdout("%*s%s %d - %s\n", level, "", verdict,
-                                           j + 1, test_title);
+                        test_printf_stdout("%*s%s %d - %s\n", level, "",
+                                           verdict, jj + 1, test_title);

                     else
                     else

-                        test_printf_stdout("%*s%s %d\n", level, "", verdict,
-                                           j + 1);
+                        test_printf_stdout("%*s%s %d - iteration %d\n", level,
+                                           "", verdict, jj + 1, j + 1);

                     test_flush_stdout();
                 }
             }
                     test_flush_stdout();
                 }
             }


@@ -212,7 +254,7 @@ int run_tests(const char *test_prog_name)
                 verdict = "not ok";
                 ++num_failed;
             }
                 verdict = "not ok";
                 ++num_failed;
             }

-            test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1,
+            test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, ii + 1,

                                all_tests[i].test_case_name);
             test_flush_stdout();
         }
                                all_tests[i].test_case_name);
             test_flush_stdout();
         }