X-Git-Url: https://git.openssl.org/?p=openssl.git;a=blobdiff_plain;f=crypto%2Fbn%2Fbn_asm.c;h=c43c91cc09f485fa615eddc61ddb81d20e792bf4;hp=a62fdcf07a2300cb3317ba7d370b37c12fcf6980;hb=55614f89f0beb53ebafbfc680cf7b4d114b44d30;hpb=fb81ac5e6be4041e59df9362cd63880b21e2cafc diff --git a/crypto/bn/bn_asm.c b/crypto/bn/bn_asm.c index a62fdcf07a..c43c91cc09 100644 --- a/crypto/bn/bn_asm.c +++ b/crypto/bn/bn_asm.c @@ -56,19 +56,26 @@ * [including the GNU Public Licence.] */ +#ifndef BN_DEBUG +# undef NDEBUG /* avoid conflicting definitions */ +# define NDEBUG +#endif + #include +#include #include "cryptlib.h" #include "bn_lcl.h" #if defined(BN_LLONG) || defined(BN_UMULT_HIGH) -BN_ULONG bn_mul_add_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w) +BN_ULONG bn_mul_add_words(BN_ULONG *rp, const BN_ULONG *ap, int num, BN_ULONG w) { BN_ULONG c1=0; - bn_check_num(num); + assert(num >= 0); if (num <= 0) return(c1); +#ifndef OPENSSL_SMALL_FOOTPRINT while (num&~3) { mul_add(rp[0],ap[0],w,c1); @@ -77,23 +84,24 @@ BN_ULONG bn_mul_add_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w) mul_add(rp[3],ap[3],w,c1); ap+=4; rp+=4; num-=4; } - if (num) +#endif + while (num) { - mul_add(rp[0],ap[0],w,c1); if (--num==0) return c1; - mul_add(rp[1],ap[1],w,c1); if (--num==0) return c1; - mul_add(rp[2],ap[2],w,c1); return c1; + mul_add(rp[0],ap[0],w,c1); + ap++; rp++; num--; } return(c1); } -BN_ULONG bn_mul_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w) +BN_ULONG bn_mul_words(BN_ULONG *rp, const BN_ULONG *ap, int num, BN_ULONG w) { BN_ULONG c1=0; - bn_check_num(num); + assert(num >= 0); if (num <= 0) return(c1); +#ifndef OPENSSL_SMALL_FOOTPRINT while (num&~3) { mul(rp[0],ap[0],w,c1); @@ -102,19 +110,21 @@ BN_ULONG bn_mul_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w) mul(rp[3],ap[3],w,c1); ap+=4; rp+=4; num-=4; } - if (num) +#endif + while (num) { - mul(rp[0],ap[0],w,c1); if (--num == 0) return c1; - mul(rp[1],ap[1],w,c1); if (--num == 0) return c1; - mul(rp[2],ap[2],w,c1); + mul(rp[0],ap[0],w,c1); + ap++; rp++; num--; } return(c1); } -void bn_sqr_words(BN_ULONG *r, BN_ULONG *a, int n) +void bn_sqr_words(BN_ULONG *r, const BN_ULONG *a, int n) { - bn_check_num(n); + assert(n >= 0); if (n <= 0) return; + +#ifndef OPENSSL_SMALL_FOOTPRINT while (n&~3) { sqr(r[0],r[1],a[0]); @@ -123,94 +133,97 @@ void bn_sqr_words(BN_ULONG *r, BN_ULONG *a, int n) sqr(r[6],r[7],a[3]); a+=4; r+=8; n-=4; } - if (n) +#endif + while (n) { - sqr(r[0],r[1],a[0]); if (--n == 0) return; - sqr(r[2],r[3],a[1]); if (--n == 0) return; - sqr(r[4],r[5],a[2]); + sqr(r[0],r[1],a[0]); + a++; r+=2; n--; } } -#else +#else /* !(defined(BN_LLONG) || defined(BN_UMULT_HIGH)) */ -BN_ULONG bn_mul_add_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w) +BN_ULONG bn_mul_add_words(BN_ULONG *rp, const BN_ULONG *ap, int num, BN_ULONG w) { BN_ULONG c=0; BN_ULONG bl,bh; - bn_check_num(num); + assert(num >= 0); if (num <= 0) return((BN_ULONG)0); bl=LBITS(w); bh=HBITS(w); - for (;;) +#ifndef OPENSSL_SMALL_FOOTPRINT + while (num&~3) { mul_add(rp[0],ap[0],bl,bh,c); - if (--num == 0) break; mul_add(rp[1],ap[1],bl,bh,c); - if (--num == 0) break; mul_add(rp[2],ap[2],bl,bh,c); - if (--num == 0) break; mul_add(rp[3],ap[3],bl,bh,c); - if (--num == 0) break; - ap+=4; - rp+=4; + ap+=4; rp+=4; num-=4; + } +#endif + while (num) + { + mul_add(rp[0],ap[0],bl,bh,c); + ap++; rp++; num--; } return(c); } -BN_ULONG bn_mul_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w) +BN_ULONG bn_mul_words(BN_ULONG *rp, const BN_ULONG *ap, int num, BN_ULONG w) { BN_ULONG carry=0; BN_ULONG bl,bh; - bn_check_num(num); + assert(num >= 0); if (num <= 0) return((BN_ULONG)0); bl=LBITS(w); bh=HBITS(w); - for (;;) +#ifndef OPENSSL_SMALL_FOOTPRINT + while (num&~3) { mul(rp[0],ap[0],bl,bh,carry); - if (--num == 0) break; mul(rp[1],ap[1],bl,bh,carry); - if (--num == 0) break; mul(rp[2],ap[2],bl,bh,carry); - if (--num == 0) break; mul(rp[3],ap[3],bl,bh,carry); - if (--num == 0) break; - ap+=4; - rp+=4; + ap+=4; rp+=4; num-=4; + } +#endif + while (num) + { + mul(rp[0],ap[0],bl,bh,carry); + ap++; rp++; num--; } return(carry); } -void bn_sqr_words(BN_ULONG *r, BN_ULONG *a, int n) +void bn_sqr_words(BN_ULONG *r, const BN_ULONG *a, int n) { - bn_check_num(n); + assert(n >= 0); if (n <= 0) return; - for (;;) + +#ifndef OPENSSL_SMALL_FOOTPRINT + while (n&~3) { sqr64(r[0],r[1],a[0]); - if (--n == 0) break; - sqr64(r[2],r[3],a[1]); - if (--n == 0) break; - sqr64(r[4],r[5],a[2]); - if (--n == 0) break; - sqr64(r[6],r[7],a[3]); - if (--n == 0) break; - - a+=4; - r+=8; + a+=4; r+=8; n-=4; + } +#endif + while (n) + { + sqr64(r[0],r[1],a[0]); + a++; r+=2; n--; } } -#endif +#endif /* !(defined(BN_LLONG) || defined(BN_UMULT_HIGH)) */ #if defined(BN_LLONG) && defined(BN_DIV2W) @@ -221,7 +234,7 @@ BN_ULONG bn_div_words(BN_ULONG h, BN_ULONG l, BN_ULONG d) #else -/* Divide h-l by d and return the result. */ +/* Divide h,l by d and return the result. */ /* I need to test this some more :-( */ BN_ULONG bn_div_words(BN_ULONG h, BN_ULONG l, BN_ULONG d) { @@ -231,13 +244,8 @@ BN_ULONG bn_div_words(BN_ULONG h, BN_ULONG l, BN_ULONG d) if (d == 0) return(BN_MASK2); i=BN_num_bits_word(d); - if ((i != BN_BITS2) && (h > (BN_ULONG)1<= d) h-=d; @@ -292,54 +300,54 @@ BN_ULONG bn_div_words(BN_ULONG h, BN_ULONG l, BN_ULONG d) ret|=q; return(ret); } -#endif +#endif /* !defined(BN_LLONG) && defined(BN_DIV2W) */ #ifdef BN_LLONG -BN_ULONG bn_add_words(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n) +BN_ULONG bn_add_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, int n) { BN_ULLONG ll=0; - bn_check_num(n); + assert(n >= 0); if (n <= 0) return((BN_ULONG)0); - for (;;) +#ifndef OPENSSL_SMALL_FOOTPRINT + while (n&~3) { ll+=(BN_ULLONG)a[0]+b[0]; r[0]=(BN_ULONG)ll&BN_MASK2; ll>>=BN_BITS2; - if (--n <= 0) break; - ll+=(BN_ULLONG)a[1]+b[1]; r[1]=(BN_ULONG)ll&BN_MASK2; ll>>=BN_BITS2; - if (--n <= 0) break; - ll+=(BN_ULLONG)a[2]+b[2]; r[2]=(BN_ULONG)ll&BN_MASK2; ll>>=BN_BITS2; - if (--n <= 0) break; - ll+=(BN_ULLONG)a[3]+b[3]; r[3]=(BN_ULONG)ll&BN_MASK2; ll>>=BN_BITS2; - if (--n <= 0) break; - - a+=4; - b+=4; - r+=4; + a+=4; b+=4; r+=4; n-=4; + } +#endif + while (n) + { + ll+=(BN_ULLONG)a[0]+b[0]; + r[0]=(BN_ULONG)ll&BN_MASK2; + ll>>=BN_BITS2; + a++; b++; r++; n--; } return((BN_ULONG)ll); } -#else -BN_ULONG bn_add_words(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n) +#else /* !BN_LLONG */ +BN_ULONG bn_add_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, int n) { BN_ULONG c,l,t; - bn_check_num(n); + assert(n >= 0); if (n <= 0) return((BN_ULONG)0); c=0; - for (;;) +#ifndef OPENSSL_SMALL_FOOTPRINT + while (n&~3) { t=a[0]; t=(t+c)&BN_MASK2; @@ -347,84 +355,89 @@ BN_ULONG bn_add_words(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n) l=(t+b[0])&BN_MASK2; c+=(l < t); r[0]=l; - if (--n <= 0) break; - t=a[1]; t=(t+c)&BN_MASK2; c=(t < c); l=(t+b[1])&BN_MASK2; c+=(l < t); r[1]=l; - if (--n <= 0) break; - t=a[2]; t=(t+c)&BN_MASK2; c=(t < c); l=(t+b[2])&BN_MASK2; c+=(l < t); r[2]=l; - if (--n <= 0) break; - t=a[3]; t=(t+c)&BN_MASK2; c=(t < c); l=(t+b[3])&BN_MASK2; c+=(l < t); r[3]=l; - if (--n <= 0) break; - - a+=4; - b+=4; - r+=4; + a+=4; b+=4; r+=4; n-=4; + } +#endif + while(n) + { + t=a[0]; + t=(t+c)&BN_MASK2; + c=(t < c); + l=(t+b[0])&BN_MASK2; + c+=(l < t); + r[0]=l; + a++; b++; r++; n--; } return((BN_ULONG)c); } -#endif +#endif /* !BN_LLONG */ -BN_ULONG bn_sub_words(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n) +BN_ULONG bn_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, int n) { BN_ULONG t1,t2; int c=0; - bn_check_num(n); + assert(n >= 0); if (n <= 0) return((BN_ULONG)0); - for (;;) +#ifndef OPENSSL_SMALL_FOOTPRINT + while (n&~3) { t1=a[0]; t2=b[0]; r[0]=(t1-t2-c)&BN_MASK2; if (t1 != t2) c=(t1 < t2); - if (--n <= 0) break; - t1=a[1]; t2=b[1]; r[1]=(t1-t2-c)&BN_MASK2; if (t1 != t2) c=(t1 < t2); - if (--n <= 0) break; - t1=a[2]; t2=b[2]; r[2]=(t1-t2-c)&BN_MASK2; if (t1 != t2) c=(t1 < t2); - if (--n <= 0) break; - t1=a[3]; t2=b[3]; r[3]=(t1-t2-c)&BN_MASK2; if (t1 != t2) c=(t1 < t2); - if (--n <= 0) break; - - a+=4; - b+=4; - r+=4; + a+=4; b+=4; r+=4; n-=4; + } +#endif + while (n) + { + t1=a[0]; t2=b[0]; + r[0]=(t1-t2-c)&BN_MASK2; + if (t1 != t2) c=(t1 < t2); + a++; b++; r++; n--; } return(c); } -#ifdef BN_MUL_COMBA +#if defined(BN_MUL_COMBA) && !defined(OPENSSL_SMALL_FOOTPRINT) #undef bn_mul_comba8 #undef bn_mul_comba4 #undef bn_sqr_comba8 #undef bn_sqr_comba4 +/* mul_add_c(a,b,c0,c1,c2) -- c+=a*b for three word number c=(c2,c1,c0) */ +/* mul_add_c2(a,b,c0,c1,c2) -- c+=2*a*b for three word number c=(c2,c1,c0) */ +/* sqr_add_c(a,i,c0,c1,c2) -- c+=a[i]^2 for three word number c=(c2,c1,c0) */ +/* sqr_add_c2(a,i,c0,c1,c2) -- c+=2*a[i]*a[j] for three word number c=(c2,c1,c0) */ + #ifdef BN_LLONG #define mul_add_c(a,b,c0,c1,c2) \ t=(BN_ULLONG)a*b; \ @@ -453,6 +466,34 @@ BN_ULONG bn_sub_words(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n) #define sqr_add_c2(a,i,j,c0,c1,c2) \ mul_add_c2((a)[i],(a)[j],c0,c1,c2) +#elif defined(BN_UMULT_LOHI) + +#define mul_add_c(a,b,c0,c1,c2) { \ + BN_ULONG ta=(a),tb=(b); \ + BN_UMULT_LOHI(t1,t2,ta,tb); \ + c0 += t1; t2 += (c0 +/* + * This is essentially reference implementation, which may or may not + * result in performance improvement. E.g. on IA-32 this routine was + * observed to give 40% faster rsa1024 private key operations and 10% + * faster rsa4096 ones, while on AMD64 it improves rsa1024 sign only + * by 10% and *worsens* rsa4096 sign by 15%. Once again, it's a + * reference implementation, one to be used as starting point for + * platform-specific assembler. Mentioned numbers apply to compiler + * generated code compiled with and without -DOPENSSL_BN_ASM_MONT and + * can vary not only from platform to platform, but even for compiler + * versions. Assembler vs. assembler improvement coefficients can + * [and are known to] differ and are to be documented elsewhere. + */ +int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np,const BN_ULONG *n0p, int num) + { + BN_ULONG c0,c1,ml,*tp,n0; +#ifdef mul64 + BN_ULONG mh; +#endif + volatile BN_ULONG *vp; + int i=0,j; + +#if 0 /* template for platform-specific implementation */ + if (ap==bp) return bn_sqr_mont(rp,ap,np,n0p,num); +#endif + vp = tp = alloca((num+2)*sizeof(BN_ULONG)); + + n0 = *n0p; + + c0 = 0; + ml = bp[0]; +#ifdef mul64 + mh = HBITS(ml); + ml = LBITS(ml); + for (j=0;j=np[num-1]) + { + c0 = bn_sub_words(rp,tp,np,num); + if (tp[num]!=0 || c0==0) + { + for(i=0;i +int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np,const BN_ULONG *n0p, int num) + { + BN_ULONG c0,c1,*tp,n0=*n0p; + volatile BN_ULONG *vp; + int i=0,j; + + vp = tp = alloca((num+2)*sizeof(BN_ULONG)); + + for(i=0;i<=num;i++) tp[i]=0; + + for(i=0;i=np[num-1]) + { + c0 = bn_sub_words(rp,tp,np,num); + if (tp[num]!=0 || c0==0) + { + for(i=0;i