Otherwise this can result in an incorrect calculation of the maximum
encoded integer length, meaning an insufficient buffer size is allocated.
Thanks to Billy Brumley for helping to track this down.
Fixes #8209
Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/8237)
const BIGNUM *sig_r, *sig_s;
BIGNUM *modified_r = NULL, *modified_s = NULL;
BIGNUM *unmodified_r = NULL, *unmodified_s = NULL;
const BIGNUM *sig_r, *sig_s;
BIGNUM *modified_r = NULL, *modified_s = NULL;
BIGNUM *unmodified_r = NULL, *unmodified_s = NULL;
- unsigned int sig_len, degree, r_len, s_len, bn_len, buf_len;
+ unsigned int sig_len, order, r_len, s_len, bn_len, buf_len;
int nid, ret = 0;
/* fill digest values with some random data */
int nid, ret = 0;
/* fill digest values with some random data */
|| !TEST_true(EC_KEY_set_group(eckey, group)))
goto builtin_err;
EC_GROUP_free(group);
|| !TEST_true(EC_KEY_set_group(eckey, group)))
goto builtin_err;
EC_GROUP_free(group);
- degree = EC_GROUP_get_degree(EC_KEY_get0_group(eckey));
+ order = EC_GROUP_order_bits(EC_KEY_get0_group(eckey));
TEST_info("testing %s", OBJ_nid2sn(nid));
TEST_info("testing %s", OBJ_nid2sn(nid));
/* Store the two BIGNUMs in raw_buf. */
r_len = BN_num_bytes(sig_r);
s_len = BN_num_bytes(sig_s);
/* Store the two BIGNUMs in raw_buf. */
r_len = BN_num_bytes(sig_r);
s_len = BN_num_bytes(sig_s);
- bn_len = (degree + 7) / 8;
+ bn_len = (order + 7) / 8;
if (!TEST_false(r_len > bn_len)
|| !TEST_false(s_len > bn_len))
goto builtin_err;
if (!TEST_false(r_len > bn_len)
|| !TEST_false(s_len > bn_len))
goto builtin_err;