X-Git-Url: https://git.openssl.org/gitweb/?a=blobdiff_plain;f=crypto%2Fbn%2Fbn_exp.c;h=baeda372cf5c0d70d552459945ccf6a0d90fdff4;hb=ad0d2579cf3a293a35a5b606afc5a97c71cf6ca7;hp=11540c6f7b0dfcad04fb48713655a8413b055651;hpb=dc434bbcb0f63e03c64be1d977fae6c9411bfc5c;p=openssl.git diff --git a/crypto/bn/bn_exp.c b/crypto/bn/bn_exp.c index 11540c6f7b..baeda372cf 100644 --- a/crypto/bn/bn_exp.c +++ b/crypto/bn/bn_exp.c @@ -56,7 +56,7 @@ * [including the GNU Public Licence.] */ /* ==================================================================== - * Copyright (c) 1998-2000 The OpenSSL Project. All rights reserved. + * Copyright (c) 1998-2005 The OpenSSL Project. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -109,56 +109,63 @@ * */ +#define OPENSSL_FIPSAPI -#include #include "cryptlib.h" #include "bn_lcl.h" -#ifdef ATALLA + +#include +#ifdef _WIN32 +# include +# ifndef alloca +# define alloca _alloca +# endif +#elif defined(__GNUC__) +# ifndef alloca +# define alloca(s) __builtin_alloca((s)) +# endif +#elif defined(__sun) # include -# include -# include -# include #endif +#undef RSAZ_ENABLED +#if defined(OPENSSL_BN_ASM_MONT) && \ + (defined(__x86_64) || defined(__x86_64__) || \ + defined(_M_AMD64) || defined(_M_X64)) +# include "rsaz_exp.h" +# define RSAZ_ENABLED +#endif -#define TABLE_SIZE 32 - -/* slow but works */ -int BN_mod_mul(BIGNUM *ret, BIGNUM *a, BIGNUM *b, const BIGNUM *m, BN_CTX *ctx) - { - BIGNUM *t; - int r=0; - - bn_check_top(a); - bn_check_top(b); - bn_check_top(m); - - BN_CTX_start(ctx); - if ((t = BN_CTX_get(ctx)) == NULL) goto err; - if (a == b) - { if (!BN_sqr(t,a,ctx)) goto err; } - else - { if (!BN_mul(t,a,b,ctx)) goto err; } - if (!BN_mod(ret,t,m,ctx)) goto err; - r=1; -err: - BN_CTX_end(ctx); - return(r); - } +#undef SPARC_T4_MONT +#if defined(OPENSSL_BN_ASM_MONT) && (defined(__sparc__) || defined(__sparc)) +# include "sparc_arch.h" +extern unsigned int OPENSSL_sparcv9cap_P[]; +# define SPARC_T4_MONT +#endif +/* maximum precomputation table size for *variable* sliding windows */ +#define TABLE_SIZE 32 /* this one works - simple but works */ -int BN_exp(BIGNUM *r, BIGNUM *a, BIGNUM *p, BN_CTX *ctx) +int BN_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) { int i,bits,ret=0; BIGNUM *v,*rr; + if (BN_get_flags(p, BN_FLG_CONSTTIME) != 0) + { + /* BN_FLG_CONSTTIME only supported by BN_mod_exp_mont() */ + BNerr(BN_F_BN_EXP,ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return -1; + } + BN_CTX_start(ctx); if ((r == a) || (r == p)) rr = BN_CTX_get(ctx); else rr = r; - if ((v = BN_CTX_get(ctx)) == NULL) goto err; + v = BN_CTX_get(ctx); + if (rr == NULL || v == NULL) goto err; if (BN_copy(v,a) == NULL) goto err; bits=BN_num_bits(p); @@ -179,179 +186,12 @@ int BN_exp(BIGNUM *r, BIGNUM *a, BIGNUM *p, BN_CTX *ctx) err: if (r != rr) BN_copy(r,rr); BN_CTX_end(ctx); + bn_check_top(r); return(ret); } -#ifdef ATALLA - -/* - * This routine will dynamically check for the existance of an Atalla AXL-200 - * SSL accelerator module. If one is found, the variable - * asi_accelerator_present is set to 1 and the function pointers - * ptr_ASI_xxxxxx above will be initialized to corresponding ASI API calls. - */ -typedef int tfnASI_GetPerformanceStatistics(int reset_flag, - unsigned int *ret_buf); -typedef int tfnASI_GetHardwareConfig(long card_num, unsigned int *ret_buf); -typedef int tfnASI_RSAPrivateKeyOpFn(RSAPrivateKey * rsaKey, - unsigned char *output, - unsigned char *input, - unsigned int modulus_len); - -static tfnASI_GetHardwareConfig *ptr_ASI_GetHardwareConfig; -static tfnASI_RSAPrivateKeyOpFn *ptr_ASI_RSAPrivateKeyOpFn; -static tfnASI_GetPerformanceStatistics *ptr_ASI_GetPerformanceStatistics; -static int asi_accelerator_present; -static int tried_atalla; - -void atalla_initialize_accelerator_handle(void) - { - void *dl_handle; - int status; - unsigned int config_buf[1024]; - static int tested; - - if(tested) - return; - - tested=1; - - bzero((void *)config_buf, 1024); - - /* - * Check to see if the library is present on the system - */ - dl_handle = dlopen("atasi.so", RTLD_NOW); - if (dl_handle == (void *) NULL) - { -/* printf("atasi.so library is not present on the system\n"); - printf("No HW acceleration available\n");*/ - return; - } - - /* - * The library is present. Now we'll check to insure that the - * LDM is up and running. First we'll get the address of the - * function in the atasi library that we need to see if the - * LDM is operating. - */ - - ptr_ASI_GetHardwareConfig = - (tfnASI_GetHardwareConfig *)dlsym(dl_handle,"ASI_GetHardwareConfig"); - - if (ptr_ASI_GetHardwareConfig) - { - /* - * We found the call, now we'll get our config - * status. If we get a non 0 result, the LDM is not - * running and we cannot use the Atalla ASI * - * library. - */ - status = (*ptr_ASI_GetHardwareConfig)(0L, config_buf); - if (status != 0) - { - printf("atasi.so library is present but not initialized\n"); - printf("No HW acceleration available\n"); - return; - } - } - else - { -/* printf("We found the library, but not the function. Very Strange!\n");*/ - return ; - } - - /* - * It looks like we have acceleration capabilities. Load up the - * pointers to our ASI API calls. - */ - ptr_ASI_RSAPrivateKeyOpFn= - (tfnASI_RSAPrivateKeyOpFn *)dlsym(dl_handle, "ASI_RSAPrivateKeyOpFn"); - if (ptr_ASI_RSAPrivateKeyOpFn == NULL) - { -/* printf("We found the library, but no RSA function. Very Strange!\n");*/ - return; - } - - ptr_ASI_GetPerformanceStatistics = - (tfnASI_GetPerformanceStatistics *)dlsym(dl_handle, "ASI_GetPerformanceStatistics"); - if (ptr_ASI_GetPerformanceStatistics == NULL) - { -/* printf("We found the library, but no stat function. Very Strange!\n");*/ - return; - } - - /* - * Indicate that acceleration is available - */ - asi_accelerator_present = 1; - -/* printf("This system has acceleration!\n");*/ - - return; - } - -/* make sure this only gets called once when bn_mod_exp calls bn_mod_exp_mont */ -int BN_mod_exp_atalla(BIGNUM *r, BIGNUM *a, const BIGNUM *p, const BIGNUM *m) - { - unsigned char *abin; - unsigned char *pbin; - unsigned char *mbin; - unsigned char *rbin; - int an,pn,mn,ret; - RSAPrivateKey keydata; - - atalla_initialize_accelerator_handle(); - if(!asi_accelerator_present) - return 0; - - -/* We should be able to run without size testing */ -# define ASIZE 128 - an=BN_num_bytes(a); - pn=BN_num_bytes(p); - mn=BN_num_bytes(m); - - if(an <= ASIZE && pn <= ASIZE && mn <= ASIZE) - { - int size=mn; - - assert(an <= mn); - abin=alloca(size); - memset(abin,'\0',mn); - BN_bn2bin(a,abin+size-an); - - pbin=alloca(pn); - BN_bn2bin(p,pbin); - - mbin=alloca(size); - memset(mbin,'\0',mn); - BN_bn2bin(m,mbin+size-mn); - - rbin=alloca(size); - - memset(&keydata,'\0',sizeof keydata); - keydata.privateExponent.data=pbin; - keydata.privateExponent.len=pn; - keydata.modulus.data=mbin; - keydata.modulus.len=size; - - ret=(*ptr_ASI_RSAPrivateKeyOpFn)(&keydata,rbin,abin,keydata.modulus.len); -/*fprintf(stderr,"!%s\n",BN_bn2hex(a));*/ - if(!ret) - { - BN_bin2bn(rbin,keydata.modulus.len,r); -/*fprintf(stderr,"?%s\n",BN_bn2hex(r));*/ - return 1; - } - } - return 0; - } -#endif /* def ATALLA */ - - -int BN_mod_exp(BIGNUM *r, BIGNUM *a, const BIGNUM *p, const BIGNUM *m, +int BN_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m, BN_CTX *ctx) { int ret; @@ -360,12 +200,39 @@ int BN_mod_exp(BIGNUM *r, BIGNUM *a, const BIGNUM *p, const BIGNUM *m, bn_check_top(p); bn_check_top(m); -#ifdef ATALLA - if(BN_mod_exp_atalla(r,a,p,m)) - return 1; -/* If it fails, try the other methods (but don't try atalla again) */ - tried_atalla=1; -#endif + /* For even modulus m = 2^k*m_odd, it might make sense to compute + * a^p mod m_odd and a^p mod 2^k separately (with Montgomery + * exponentiation for the odd part), using appropriate exponent + * reductions, and combine the results using the CRT. + * + * For now, we use Montgomery only if the modulus is odd; otherwise, + * exponentiation using the reciprocal-based quick remaindering + * algorithm is used. + * + * (Timing obtained with expspeed.c [computations a^p mod m + * where a, p, m are of the same length: 256, 512, 1024, 2048, + * 4096, 8192 bits], compared to the running time of the + * standard algorithm: + * + * BN_mod_exp_mont 33 .. 40 % [AMD K6-2, Linux, debug configuration] + * 55 .. 77 % [UltraSparc processor, but + * debug-solaris-sparcv8-gcc conf.] + * + * BN_mod_exp_recp 50 .. 70 % [AMD K6-2, Linux, debug configuration] + * 62 .. 118 % [UltraSparc, debug-solaris-sparcv8-gcc] + * + * On the Sparc, BN_mod_exp_recp was faster than BN_mod_exp_mont + * at 2048 and more bits, but at 512 and 1024 bits, it was + * slower even than the standard algorithm! + * + * "Real" timings [linux-elf, solaris-sparcv9-gcc configurations] + * should be obtained when the new Montgomery reduction code + * has been integrated into OpenSSL.) + */ + +#define MONT_MUL_MOD +#define MONT_EXP_WORD +#define RECP_MUL_MOD #ifdef MONT_MUL_MOD /* I have finally been able to take out this pre-condition of @@ -376,12 +243,14 @@ int BN_mod_exp(BIGNUM *r, BIGNUM *a, const BIGNUM *p, const BIGNUM *m, if (BN_is_odd(m)) { - if (a->top == 1) +# ifdef MONT_EXP_WORD + if (a->top == 1 && !a->neg && (BN_get_flags(p, BN_FLG_CONSTTIME) == 0)) { BN_ULONG A = a->d[0]; ret=BN_mod_exp_mont_word(r,A,p,m,ctx,NULL); } else +# endif ret=BN_mod_exp_mont(r,a,p,m,ctx,NULL); } else @@ -392,10 +261,7 @@ int BN_mod_exp(BIGNUM *r, BIGNUM *a, const BIGNUM *p, const BIGNUM *m, { ret=BN_mod_exp_simple(r,a,p,m,ctx); } #endif -#ifdef ATALLA - tried_atalla=0; -#endif - + bn_check_top(r); return(ret); } @@ -404,43 +270,66 @@ int BN_mod_exp_recp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const BIGNUM *m, BN_CTX *ctx) { int i,j,bits,ret=0,wstart,wend,window,wvalue; - int start=1,ts=0; + int start=1; BIGNUM *aa; - BIGNUM val[TABLE_SIZE]; + /* Table of variables obtained from 'ctx' */ + BIGNUM *val[TABLE_SIZE]; BN_RECP_CTX recp; + if (BN_get_flags(p, BN_FLG_CONSTTIME) != 0) + { + /* BN_FLG_CONSTTIME only supported by BN_mod_exp_mont() */ + BNerr(BN_F_BN_MOD_EXP_RECP,ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return -1; + } + bits=BN_num_bits(p); if (bits == 0) { - BN_one(r); - return(1); + ret = BN_one(r); + return ret; } BN_CTX_start(ctx); - if ((aa = BN_CTX_get(ctx)) == NULL) goto err; + aa = BN_CTX_get(ctx); + val[0] = BN_CTX_get(ctx); + if(!aa || !val[0]) goto err; BN_RECP_CTX_init(&recp); - if (BN_RECP_CTX_set(&recp,m,ctx) <= 0) goto err; - - BN_init(&(val[0])); - ts=1; + if (m->neg) + { + /* ignore sign of 'm' */ + if (!BN_copy(aa, m)) goto err; + aa->neg = 0; + if (BN_RECP_CTX_set(&recp,aa,ctx) <= 0) goto err; + } + else + { + if (BN_RECP_CTX_set(&recp,m,ctx) <= 0) goto err; + } - if (!BN_mod(&(val[0]),a,m,ctx)) goto err; /* 1 */ + if (!BN_nnmod(val[0],a,m,ctx)) goto err; /* 1 */ + if (BN_is_zero(val[0])) + { + BN_zero(r); + ret = 1; + goto err; + } window = BN_window_bits_for_exponent_size(bits); if (window > 1) { - if (!BN_mod_mul_reciprocal(aa,&(val[0]),&(val[0]),&recp,ctx)) + if (!BN_mod_mul_reciprocal(aa,val[0],val[0],&recp,ctx)) goto err; /* 2 */ j=1<<(window-1); for (i=1; i>1]),&recp,ctx)) + if (!BN_mod_mul_reciprocal(r,r,val[wvalue>>1],&recp,ctx)) goto err; /* move the 'window' down further */ @@ -504,34 +393,33 @@ int BN_mod_exp_recp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, ret=1; err: BN_CTX_end(ctx); - for (i=0; id[0] & 1)) + if (!BN_is_odd(m)) { BNerr(BN_F_BN_MOD_EXP_MONT,BN_R_CALLED_WITH_EVEN_MODULUS); return(0); @@ -539,13 +427,15 @@ int BN_mod_exp_mont(BIGNUM *rr, BIGNUM *a, const BIGNUM *p, bits=BN_num_bits(p); if (bits == 0) { - BN_one(rr); - return(1); + ret = BN_one(rr); + return ret; } + BN_CTX_start(ctx); d = BN_CTX_get(ctx); r = BN_CTX_get(ctx); - if (d == NULL || r == NULL) goto err; + val[0] = BN_CTX_get(ctx); + if (!d || !r || !val[0]) goto err; /* If this is not done, things will break in the montgomery * part */ @@ -558,30 +448,34 @@ int BN_mod_exp_mont(BIGNUM *rr, BIGNUM *a, const BIGNUM *p, if (!BN_MONT_CTX_set(mont,m,ctx)) goto err; } - BN_init(&val[0]); - ts=1; - if (BN_ucmp(a,m) >= 0) + if (a->neg || BN_ucmp(a,m) >= 0) { - if (!BN_mod(&(val[0]),a,m,ctx)) + if (!BN_nnmod(val[0],a,m,ctx)) goto err; - aa= &(val[0]); + aa= val[0]; } else aa=a; - if (!BN_to_montgomery(&(val[0]),aa,mont,ctx)) goto err; /* 1 */ + if (BN_is_zero(aa)) + { + BN_zero(rr); + ret = 1; + goto err; + } + if (!BN_to_montgomery(val[0],aa,mont,ctx)) goto err; /* 1 */ window = BN_window_bits_for_exponent_size(bits); if (window > 1) { - if (!BN_mod_mul_montgomery(d,&(val[0]),&(val[0]),mont,ctx)) goto err; /* 2 */ + if (!BN_mod_mul_montgomery(d,val[0],val[0],mont,ctx)) goto err; /* 2 */ j=1<<(window-1); for (i=1; itop; /* borrow j */ + if (m->d[j-1] & (((BN_ULONG)1)<<(BN_BITS2-1))) + { + if (bn_wexpand(r,j) == NULL) goto err; + /* 2^(top*BN_BITS2) - m */ + r->d[0] = (0-m->d[0])&BN_MASK2; + for(i=1;id[i] = (~m->d[i])&BN_MASK2; + r->top = j; + } + else +#endif if (!BN_to_montgomery(r,BN_value_one(),mont,ctx)) goto err; for (;;) { @@ -634,7 +540,7 @@ int BN_mod_exp_mont(BIGNUM *rr, BIGNUM *a, const BIGNUM *p, } /* wvalue will be an odd number < 2^window */ - if (!BN_mod_mul_montgomery(r,r,&(val[wvalue>>1]),mont,ctx)) + if (!BN_mod_mul_montgomery(r,r,val[wvalue>>1],mont,ctx)) goto err; /* move the 'window' down further */ @@ -643,13 +549,547 @@ int BN_mod_exp_mont(BIGNUM *rr, BIGNUM *a, const BIGNUM *p, start=0; if (wstart < 0) break; } - BN_from_montgomery(rr,r,mont,ctx); +#if defined(SPARC_T4_MONT) + if (OPENSSL_sparcv9cap_P[0]&(SPARCV9_VIS3|SPARCV9_PREFER_FPU)) + { + j = mont->N.top; /* borrow j */ + val[0]->d[0] = 1; /* borrow val[0] */ + for (i=1;id[i] = 0; + val[0]->top = j; + if (!BN_mod_mul_montgomery(rr,r,val[0],mont,ctx)) goto err; + } + else +#endif + if (!BN_from_montgomery(rr,r,mont,ctx)) goto err; + ret=1; +err: + if ((in_mont == NULL) && (mont != NULL)) BN_MONT_CTX_free(mont); + BN_CTX_end(ctx); + bn_check_top(rr); + return(ret); + } + +#if defined(SPARC_T4_MONT) +static BN_ULONG bn_get_bits(const BIGNUM *a, int bitpos) + { + BN_ULONG ret=0; + int wordpos; + + wordpos = bitpos/BN_BITS2; + bitpos %= BN_BITS2; + if (wordpos>=0 && wordpos < a->top) + { + ret = a->d[wordpos]&BN_MASK2; + if (bitpos) + { + ret >>= bitpos; + if (++wordpos < a->top) + ret |= a->d[wordpos]<<(BN_BITS2-bitpos); + } + } + + return ret&BN_MASK2; +} +#endif + +/* BN_mod_exp_mont_consttime() stores the precomputed powers in a specific layout + * so that accessing any of these table values shows the same access pattern as far + * as cache lines are concerned. The following functions are used to transfer a BIGNUM + * from/to that table. */ + +static int MOD_EXP_CTIME_COPY_TO_PREBUF(const BIGNUM *b, int top, unsigned char *buf, int idx, int width) + { + size_t i, j; + + if (top > b->top) + top = b->top; /* this works because 'buf' is explicitly zeroed */ + for (i = 0, j=idx; i < top * sizeof b->d[0]; i++, j+=width) + { + buf[j] = ((unsigned char*)b->d)[i]; + } + + return 1; + } + +static int MOD_EXP_CTIME_COPY_FROM_PREBUF(BIGNUM *b, int top, unsigned char *buf, int idx, int width) + { + size_t i, j; + + if (bn_wexpand(b, top) == NULL) + return 0; + + for (i=0, j=idx; i < top * sizeof b->d[0]; i++, j+=width) + { + ((unsigned char*)b->d)[i] = buf[j]; + } + + b->top = top; + bn_correct_top(b); + return 1; + } + +/* Given a pointer value, compute the next address that is a cache line multiple. */ +#define MOD_EXP_CTIME_ALIGN(x_) \ + ((unsigned char*)(x_) + (MOD_EXP_CTIME_MIN_CACHE_LINE_WIDTH - (((size_t)(x_)) & (MOD_EXP_CTIME_MIN_CACHE_LINE_MASK)))) + +/* This variant of BN_mod_exp_mont() uses fixed windows and the special + * precomputation memory layout to limit data-dependency to a minimum + * to protect secret exponents (cf. the hyper-threading timing attacks + * pointed out by Colin Percival, + * http://www.daemonology.net/hyperthreading-considered-harmful/) + */ +int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p, + const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *in_mont) + { + int i,bits,ret=0,window,wvalue; + int top; + BN_MONT_CTX *mont=NULL; + + int numPowers; + unsigned char *powerbufFree=NULL; + int powerbufLen = 0; + unsigned char *powerbuf=NULL; + BIGNUM tmp, am; +#if defined(SPARC_T4_MONT) + unsigned int t4=0; +#endif + + bn_check_top(a); + bn_check_top(p); + bn_check_top(m); + + top = m->top; + + if (!(m->d[0] & 1)) + { + BNerr(BN_F_BN_MOD_EXP_MONT_CONSTTIME,BN_R_CALLED_WITH_EVEN_MODULUS); + return(0); + } + bits=BN_num_bits(p); + if (bits == 0) + { + ret = BN_one(rr); + return ret; + } + + BN_CTX_start(ctx); + + /* Allocate a montgomery context if it was not supplied by the caller. + * If this is not done, things will break in the montgomery part. + */ + if (in_mont != NULL) + mont=in_mont; + else + { + if ((mont=BN_MONT_CTX_new()) == NULL) goto err; + if (!BN_MONT_CTX_set(mont,m,ctx)) goto err; + } + +#ifdef RSAZ_ENABLED + /* + * If the size of the operands allow it, perform the optimized + * RSAZ exponentiation. For further information see + * crypto/bn/rsaz_exp.c and accompanying assembly modules. + */ + if (((OPENSSL_ia32cap_P[2]&0x80100) != 0x80100) /* check for MULX/AD*X */ + && (16 == a->top) && (16 == p->top) && (BN_num_bits(m) == 1024) + && rsaz_avx2_eligible()) + { + if (NULL == bn_wexpand(rr, 16)) goto err; + RSAZ_1024_mod_exp_avx2(rr->d, a->d, p->d, m->d, mont->RR.d, mont->n0[0]); + rr->top = 16; + rr->neg = 0; + bn_correct_top(rr); + ret = 1; + goto err; + } + else if ((8 == a->top) && (8 == p->top) && (BN_num_bits(m) == 512)) + { + if (NULL == bn_wexpand(rr,8)) goto err; + RSAZ_512_mod_exp(rr->d, a->d, p->d, m->d, mont->n0[0], mont->RR.d); + rr->top = 8; + rr->neg = 0; + bn_correct_top(rr); + ret = 1; + goto err; + } +#endif + + /* Get the window size to use with size of p. */ + window = BN_window_bits_for_ctime_exponent_size(bits); +#if defined(SPARC_T4_MONT) + if (window>=5 && (top&15)==0 && top<=64 && + (OPENSSL_sparcv9cap_P[1]&(CFR_MONTMUL|CFR_MONTSQR))== + (CFR_MONTMUL|CFR_MONTSQR) && + (t4=OPENSSL_sparcv9cap_P[0])) + window=5; + else +#endif +#if defined(OPENSSL_BN_ASM_MONT5) + if (window>=5) + { + window=5; /* ~5% improvement for RSA2048 sign, and even for RSA4096 */ + if ((top&7)==0) powerbufLen += 2*top*sizeof(m->d[0]); + } +#endif + (void)0; + + /* Allocate a buffer large enough to hold all of the pre-computed + * powers of am, am itself and tmp. + */ + numPowers = 1 << window; + powerbufLen += sizeof(m->d[0])*(top*numPowers + + ((2*top)>numPowers?(2*top):numPowers)); +#ifdef alloca + if (powerbufLen < 3072) + powerbufFree = alloca(powerbufLen+MOD_EXP_CTIME_MIN_CACHE_LINE_WIDTH); + else +#endif + if ((powerbufFree=(unsigned char*)OPENSSL_malloc(powerbufLen+MOD_EXP_CTIME_MIN_CACHE_LINE_WIDTH)) == NULL) + goto err; + + powerbuf = MOD_EXP_CTIME_ALIGN(powerbufFree); + memset(powerbuf, 0, powerbufLen); + +#ifdef alloca + if (powerbufLen < 3072) + powerbufFree = NULL; +#endif + + /* lay down tmp and am right after powers table */ + tmp.d = (BN_ULONG *)(powerbuf + sizeof(m->d[0])*top*numPowers); + am.d = tmp.d + top; + tmp.top = am.top = 0; + tmp.dmax = am.dmax = top; + tmp.neg = am.neg = 0; + tmp.flags = am.flags = BN_FLG_STATIC_DATA; + + /* prepare a^0 in Montgomery domain */ +#if 1 /* by Shay Gueron's suggestion */ + if (m->d[top-1] & (((BN_ULONG)1)<<(BN_BITS2-1))) + { + /* 2^(top*BN_BITS2) - m */ + tmp.d[0] = (0-m->d[0])&BN_MASK2; + for (i=1;id[i])&BN_MASK2; + tmp.top = top; + } + else +#endif + if (!BN_to_montgomery(&tmp,BN_value_one(),mont,ctx)) goto err; + + /* prepare a^1 in Montgomery domain */ + if (a->neg || BN_ucmp(a,m) >= 0) + { + if (!BN_mod(&am,a,m,ctx)) goto err; + if (!BN_to_montgomery(&am,&am,mont,ctx)) goto err; + } + else if (!BN_to_montgomery(&am,a,mont,ctx)) goto err; + +#if defined(SPARC_T4_MONT) + if (t4) + { + typedef int (*bn_pwr5_mont_f)(BN_ULONG *tp,const BN_ULONG *np, + const BN_ULONG *n0,const void *table,int power,int bits); + int bn_pwr5_mont_t4_8(BN_ULONG *tp,const BN_ULONG *np, + const BN_ULONG *n0,const void *table,int power,int bits); + int bn_pwr5_mont_t4_16(BN_ULONG *tp,const BN_ULONG *np, + const BN_ULONG *n0,const void *table,int power,int bits); + int bn_pwr5_mont_t4_24(BN_ULONG *tp,const BN_ULONG *np, + const BN_ULONG *n0,const void *table,int power,int bits); + int bn_pwr5_mont_t4_32(BN_ULONG *tp,const BN_ULONG *np, + const BN_ULONG *n0,const void *table,int power,int bits); + static const bn_pwr5_mont_f pwr5_funcs[4] = { + bn_pwr5_mont_t4_8, bn_pwr5_mont_t4_16, + bn_pwr5_mont_t4_24, bn_pwr5_mont_t4_32 }; + bn_pwr5_mont_f pwr5_worker = pwr5_funcs[top/16-1]; + + typedef int (*bn_mul_mont_f)(BN_ULONG *rp,const BN_ULONG *ap, + const void *bp,const BN_ULONG *np,const BN_ULONG *n0); + int bn_mul_mont_t4_8(BN_ULONG *rp,const BN_ULONG *ap, + const void *bp,const BN_ULONG *np,const BN_ULONG *n0); + int bn_mul_mont_t4_16(BN_ULONG *rp,const BN_ULONG *ap, + const void *bp,const BN_ULONG *np,const BN_ULONG *n0); + int bn_mul_mont_t4_24(BN_ULONG *rp,const BN_ULONG *ap, + const void *bp,const BN_ULONG *np,const BN_ULONG *n0); + int bn_mul_mont_t4_32(BN_ULONG *rp,const BN_ULONG *ap, + const void *bp,const BN_ULONG *np,const BN_ULONG *n0); + static const bn_mul_mont_f mul_funcs[4] = { + bn_mul_mont_t4_8, bn_mul_mont_t4_16, + bn_mul_mont_t4_24, bn_mul_mont_t4_32 }; + bn_mul_mont_f mul_worker = mul_funcs[top/16-1]; + + void bn_mul_mont_vis3(BN_ULONG *rp,const BN_ULONG *ap, + const void *bp,const BN_ULONG *np, + const BN_ULONG *n0,int num); + void bn_mul_mont_t4(BN_ULONG *rp,const BN_ULONG *ap, + const void *bp,const BN_ULONG *np, + const BN_ULONG *n0,int num); + void bn_mul_mont_gather5_t4(BN_ULONG *rp,const BN_ULONG *ap, + const void *table,const BN_ULONG *np, + const BN_ULONG *n0,int num,int power); + void bn_flip_n_scatter5_t4(const BN_ULONG *inp,size_t num, + void *table,size_t power); + void bn_gather5_t4(BN_ULONG *out,size_t num, + void *table,size_t power); + void bn_flip_t4(BN_ULONG *dst,BN_ULONG *src,size_t num); + + BN_ULONG *np=mont->N.d, *n0=mont->n0; + int stride = 5*(6-(top/16-1)); /* multiple of 5, but less than 32 */ + + /* BN_to_montgomery can contaminate words above .top + * [in BN_DEBUG[_DEBUG] build]... */ + for (i=am.top; iN.d,top); + + bits--; + for (wvalue=0, i=bits%5; i>=0; i--,bits--) + wvalue = (wvalue<<1)+BN_is_bit_set(p,bits); + bn_gather5_t4(tmp.d,top,powerbuf,wvalue); + + /* Scan the exponent one window at a time starting from the most + * significant bits. + */ + while (bits >= 0) + { + if (bits < stride) stride = bits+1; + bits -= stride; + wvalue = bn_get_bits(p,bits+1); + + if ((*pwr5_worker)(tmp.d,np,n0,powerbuf,wvalue,stride)) continue; + /* retry once and fall back */ + if ((*pwr5_worker)(tmp.d,np,n0,powerbuf,wvalue,stride)) continue; + + bits += stride-5; + wvalue >>= stride-5; + wvalue &= 31; + bn_mul_mont_t4(tmp.d,tmp.d,tmp.d,np,n0,top); + bn_mul_mont_t4(tmp.d,tmp.d,tmp.d,np,n0,top); + bn_mul_mont_t4(tmp.d,tmp.d,tmp.d,np,n0,top); + bn_mul_mont_t4(tmp.d,tmp.d,tmp.d,np,n0,top); + bn_mul_mont_t4(tmp.d,tmp.d,tmp.d,np,n0,top); + bn_mul_mont_gather5_t4(tmp.d,tmp.d,powerbuf,np,n0,top,wvalue); + } + + bn_flip_t4(tmp.d,tmp.d,top); + top *= 2; + /* back to 32-bit domain */ + tmp.top=top; + bn_correct_top(&tmp); + OPENSSL_cleanse(np,top*sizeof(BN_ULONG)); + } + else +#endif +#if defined(OPENSSL_BN_ASM_MONT5) + /* This optimization uses ideas from http://eprint.iacr.org/2011/239, + * specifically optimization of cache-timing attack countermeasures + * and pre-computation optimization. */ + + /* Dedicated window==4 case improves 512-bit RSA sign by ~15%, but as + * 512-bit RSA is hardly relevant, we omit it to spare size... */ + if (window==5) + { + void bn_mul_mont_gather5(BN_ULONG *rp,const BN_ULONG *ap, + const void *table,const BN_ULONG *np, + const BN_ULONG *n0,int num,int power); + void bn_scatter5(const BN_ULONG *inp,size_t num, + void *table,size_t power); + void bn_gather5(BN_ULONG *out,size_t num, + void *table,size_t power); + void bn_power5(BN_ULONG *rp,const BN_ULONG *ap, + const void *table,const BN_ULONG *np, + const BN_ULONG *n0,int num,int power); + int bn_get_bits5(const BN_ULONG *ap,int off); + int bn_from_montgomery(BN_ULONG *rp,const BN_ULONG *ap, + const BN_ULONG *not_used,const BN_ULONG *np, + const BN_ULONG *n0,int num); + + BN_ULONG *np=mont->N.d, *n0=mont->n0, *np2; + + /* BN_to_montgomery can contaminate words above .top + * [in BN_DEBUG[_DEBUG] build]... */ + for (i=am.top; i=0; i--,bits--) + wvalue = (wvalue<<1)+BN_is_bit_set(p,bits); + bn_gather5(tmp.d,top,powerbuf,wvalue); + + /* Scan the exponent one window at a time starting from the most + * significant bits. + */ + if (top&7) + while (bits >= 0) + { + for (wvalue=0, i=0; i<5; i++,bits--) + wvalue = (wvalue<<1)+BN_is_bit_set(p,bits); + + bn_mul_mont(tmp.d,tmp.d,tmp.d,np,n0,top); + bn_mul_mont(tmp.d,tmp.d,tmp.d,np,n0,top); + bn_mul_mont(tmp.d,tmp.d,tmp.d,np,n0,top); + bn_mul_mont(tmp.d,tmp.d,tmp.d,np,n0,top); + bn_mul_mont(tmp.d,tmp.d,tmp.d,np,n0,top); + bn_mul_mont_gather5(tmp.d,tmp.d,powerbuf,np,n0,top,wvalue); + } + else + { + while (bits >= 0) + { + wvalue = bn_get_bits5(p->d,bits-4); + bits-=5; + bn_power5(tmp.d,tmp.d,powerbuf,np2,n0,top,wvalue); + } + } + + ret=bn_from_montgomery(tmp.d,tmp.d,NULL,np2,n0,top); + tmp.top=top; + bn_correct_top(&tmp); + if (ret) + { + if (!BN_copy(rr,&tmp)) ret=0; + goto err; /* non-zero ret means it's not error */ + } + } + else +#endif + { + if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&tmp, top, powerbuf, 0, numPowers)) goto err; + if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&am, top, powerbuf, 1, numPowers)) goto err; + + /* If the window size is greater than 1, then calculate + * val[i=2..2^winsize-1]. Powers are computed as a*a^(i-1) + * (even powers could instead be computed as (a^(i/2))^2 + * to use the slight performance advantage of sqr over mul). + */ + if (window > 1) + { + if (!BN_mod_mul_montgomery(&tmp,&am,&am,mont,ctx)) goto err; + if (!MOD_EXP_CTIME_COPY_TO_PREBUF(&tmp, top, powerbuf, 2, numPowers)) goto err; + for (i=3; i=0; i--,bits--) + wvalue = (wvalue<<1)+BN_is_bit_set(p,bits); + if (!MOD_EXP_CTIME_COPY_FROM_PREBUF(&tmp,top,powerbuf,wvalue,numPowers)) goto err; + + /* Scan the exponent one window at a time starting from the most + * significant bits. + */ + while (bits >= 0) + { + wvalue=0; /* The 'value' of the window */ + + /* Scan the window, squaring the result as we go */ + for (i=0; i= 0 ? \ - (BN_mod(t, r, m, ctx) && (swap_tmp = r, r = t, t = swap_tmp, 1)) : \ - 1)) + (/* BN_ucmp(r, (m)) < 0 ? 1 :*/ \ + (BN_mod(t, r, m, ctx) && (swap_tmp = r, r = t, t = swap_tmp, 1)))) + /* BN_MOD_MUL_WORD is only used with 'w' large, + * so the BN_ucmp test is probably more overhead + * than always using BN_mod (which uses BN_copy if + * a similar test returns true). */ + /* We can use BN_mod and do not need BN_nnmod because our + * accumulator is never negative (the result of BN_mod does + * not depend on the sign of the modulus). + */ +#define BN_TO_MONTGOMERY_WORD(r, w, mont) \ + (BN_set_word(r, (w)) && BN_to_montgomery(r, r, (mont), ctx)) + + if (BN_get_flags(p, BN_FLG_CONSTTIME) != 0) + { + /* BN_FLG_CONSTTIME only supported by BN_mod_exp_mont() */ + BNerr(BN_F_BN_MOD_EXP_MONT_WORD,ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return -1; + } bn_check_top(p); bn_check_top(m); - if (!(m->d[0] & 1)) + if (!BN_is_odd(m)) { BNerr(BN_F_BN_MOD_EXP_MONT_WORD,BN_R_CALLED_WITH_EVEN_MODULUS); return(0); } + if (m->top == 1) + a %= m->d[0]; /* make sure that 'a' is reduced */ + bits = BN_num_bits(p); if (bits == 0) { - BN_one(rr); - return(1); + /* x**0 mod 1 is still zero. */ + if (BN_is_one(m)) + { + ret = 1; + BN_zero(rr); + } + else + ret = BN_one(rr); + return ret; + } + if (a == 0) + { + BN_zero(rr); + ret = 1; + return ret; } + BN_CTX_start(ctx); d = BN_CTX_get(ctx); r = BN_CTX_get(ctx); t = BN_CTX_get(ctx); if (d == NULL || r == NULL || t == NULL) goto err; -#ifdef ATALLA - if (!tried_atalla) - { - BN_set_word(t, a); - if (BN_mod_exp_word_atalla(rr, t, p, m)) - return 1; - } -/* If it fails, try the other methods */ -#endif - if (in_mont != NULL) mont=in_mont; else @@ -705,7 +1169,7 @@ int BN_mod_exp_mont_word(BIGNUM *rr, BN_ULONG a, const BIGNUM *p, if (!BN_MONT_CTX_set(mont, m, ctx)) goto err; } - if (!BN_to_montgomery(r, BN_value_one(), mont, ctx)) goto err; + r_is_one = 1; /* except for Montgomery factor */ /* bits-1 >= 0 */ @@ -717,13 +1181,22 @@ int BN_mod_exp_mont_word(BIGNUM *rr, BN_ULONG a, const BIGNUM *p, next_w = w*w; if ((next_w/w) != w) /* overflow */ { - if (!BN_MOD_MUL_WORD(r, w, m)) - goto err; + if (r_is_one) + { + if (!BN_TO_MONTGOMERY_WORD(r, w, mont)) goto err; + r_is_one = 0; + } + else + { + if (!BN_MOD_MUL_WORD(r, w, m)) goto err; + } next_w = 1; } w = next_w; - if (!BN_mod_mul_montgomery(r, r, r, mont, ctx)) - goto err; + if (!r_is_one) + { + if (!BN_mod_mul_montgomery(r, r, r, mont, ctx)) goto err; + } /* Second, multiply r*w by 'a' if exponent bit is set. */ if (BN_is_bit_set(p, b)) @@ -731,66 +1204,102 @@ int BN_mod_exp_mont_word(BIGNUM *rr, BN_ULONG a, const BIGNUM *p, next_w = w*a; if ((next_w/a) != w) /* overflow */ { - if (!BN_MOD_MUL_WORD(r, w, m)) - goto err; + if (r_is_one) + { + if (!BN_TO_MONTGOMERY_WORD(r, w, mont)) goto err; + r_is_one = 0; + } + else + { + if (!BN_MOD_MUL_WORD(r, w, m)) goto err; + } next_w = a; } w = next_w; } } + /* Finally, set r:=r*w. */ if (w != 1) { - if (!BN_MOD_MUL_WORD(r, w, m)) - goto err; + if (r_is_one) + { + if (!BN_TO_MONTGOMERY_WORD(r, w, mont)) goto err; + r_is_one = 0; + } + else + { + if (!BN_MOD_MUL_WORD(r, w, m)) goto err; + } } - BN_from_montgomery(rr, r, mont, ctx); + if (r_is_one) /* can happen only if a == 1*/ + { + if (!BN_one(rr)) goto err; + } + else + { + if (!BN_from_montgomery(rr, r, mont, ctx)) goto err; + } ret = 1; err: if ((in_mont == NULL) && (mont != NULL)) BN_MONT_CTX_free(mont); BN_CTX_end(ctx); + bn_check_top(rr); return(ret); } /* The old fallback, simple version :-) */ -int BN_mod_exp_simple(BIGNUM *r, BIGNUM *a, BIGNUM *p, BIGNUM *m, - BN_CTX *ctx) +int BN_mod_exp_simple(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, + const BIGNUM *m, BN_CTX *ctx) { - int i,j,bits,ret=0,wstart,wend,window,wvalue,ts=0; + int i,j,bits,ret=0,wstart,wend,window,wvalue; int start=1; BIGNUM *d; - BIGNUM val[TABLE_SIZE]; + /* Table of variables obtained from 'ctx' */ + BIGNUM *val[TABLE_SIZE]; + + if (BN_get_flags(p, BN_FLG_CONSTTIME) != 0) + { + /* BN_FLG_CONSTTIME only supported by BN_mod_exp_mont() */ + BNerr(BN_F_BN_MOD_EXP_SIMPLE,ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return -1; + } bits=BN_num_bits(p); if (bits == 0) { - BN_one(r); - return(1); + ret = BN_one(r); + return ret; } BN_CTX_start(ctx); - if ((d = BN_CTX_get(ctx)) == NULL) goto err; + d = BN_CTX_get(ctx); + val[0] = BN_CTX_get(ctx); + if(!d || !val[0]) goto err; - BN_init(&(val[0])); - ts=1; - if (!BN_mod(&(val[0]),a,m,ctx)) goto err; /* 1 */ + if (!BN_nnmod(val[0],a,m,ctx)) goto err; /* 1 */ + if (BN_is_zero(val[0])) + { + BN_zero(r); + ret = 1; + goto err; + } window = BN_window_bits_for_exponent_size(bits); if (window > 1) { - if (!BN_mod_mul(d,&(val[0]),&(val[0]),m,ctx)) + if (!BN_mod_mul(d,val[0],val[0],m,ctx)) goto err; /* 2 */ j=1<<(window-1); for (i=1; i>1]),m,ctx)) + if (!BN_mod_mul(r,r,val[wvalue>>1],m,ctx)) goto err; /* move the 'window' down further */ @@ -854,8 +1363,6 @@ int BN_mod_exp_simple(BIGNUM *r, BIGNUM *a, BIGNUM *p, BIGNUM *m, ret=1; err: BN_CTX_end(ctx); - for (i=0; i