Previously, BN_is_prime_fasttest_ex, when doing trial-division, would
check whether the candidate is a multiple of a number of small primes
and, if so, reject it. However, three is a multiple of three yet is
still a prime number.
This change accepts small primes as prime when doing trial-division.
Reviewed-by: Kurt Roeckx <kurt@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3264)
if (mod == (BN_ULONG)-1)
goto err;
if (mod == 0)
if (mod == (BN_ULONG)-1)
goto err;
if (mod == 0)
+ return BN_is_word(a, primes[i]);
}
if (!BN_GENCB_call(cb, 1, -1))
goto err;
}
if (!BN_GENCB_call(cb, 1, -1))
goto err;
+static int test_3_is_prime()
+{
+ int ret = 0;
+ BIGNUM *r = BN_new();
+
+ /* For a long time, small primes were not considered prime when
+ * do_trial_division was set. */
+ if (r == NULL ||
+ !BN_set_word(r, 3) ||
+ BN_is_prime_fasttest_ex(r, 3 /* nchecks */, ctx,
+ 0 /* do_trial_division */, NULL) != 1 ||
+ BN_is_prime_fasttest_ex(r, 3 /* nchecks */, ctx,
+ 1 /* do_trial_division */, NULL) != 1) {
+ goto err;
+ }
+
+ ret = 1;
+
+err:
+ BN_free(r);
+ return ret;
+}
+
/* Delete leading and trailing spaces from a string */
static char *strip_spaces(char *p)
/* Delete leading and trailing spaces from a string */
static char *strip_spaces(char *p)
ADD_TEST(test_gf2m_modsqrt);
ADD_TEST(test_gf2m_modsolvequad);
#endif
ADD_TEST(test_gf2m_modsqrt);
ADD_TEST(test_gf2m_modsolvequad);
#endif
+ ADD_TEST(test_3_is_prime);
ADD_TEST(file_tests);
RAND_seed(rnd_seed, sizeof rnd_seed);
ADD_TEST(file_tests);
RAND_seed(rnd_seed, sizeof rnd_seed);