Unobtrusive backport of 32-bit x86 Montgomery improvements from 0.9.9-dev:
[openssl.git] / crypto / bn / bn_mont.c
index 23e4ba5140255ed2c7c3bde883283a24d84ad189..e17c697e392538e365c279fb0acb75ceb814aaf8 100644 (file)
 
 #define MONT_WORD /* use the faster word-based algorithm */
 
+#if defined(MONT_WORD) && defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32)
+static int BN_from_montgomery_word(BIGNUM *ret, BIGNUM *r, BN_MONT_CTX *mont);
+#endif
+
+
+
 int BN_mod_mul_montgomery(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
                          BN_MONT_CTX *mont, BN_CTX *ctx)
        {
@@ -133,7 +139,11 @@ int BN_mod_mul_montgomery(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
        if (num>1 && a->top==num && b->top==num)
                {
                if (bn_wexpand(r,num) == NULL) return(0);
+#if defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32) /* non-default compile */
+               if (bn_mul_mont(r->d,a->d,b->d,mont->N.d,mont->n0,num))
+#else
                if (bn_mul_mont(r->d,a->d,b->d,mont->N.d,&mont->n0,num))
+#endif
                        {
                        r->neg = a->neg^b->neg;
                        r->top = num;
@@ -157,7 +167,11 @@ int BN_mod_mul_montgomery(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
                if (!BN_mul(tmp,a,b,ctx)) goto err;
                }
        /* reduce from aRR to aR */
+#if defined(MONT_WORD) && defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32)
+       if (!BN_from_montgomery_word(r,tmp,mont)) goto err;
+#else
        if (!BN_from_montgomery(r,tmp,mont,ctx)) goto err;
+#endif
        bn_check_top(r);
        ret=1;
 err:
@@ -165,6 +179,145 @@ err:
        return(ret);
        }
 
+#if defined(MONT_WORD) && defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32)
+static int BN_from_montgomery_word(BIGNUM *ret, BIGNUM *r, BN_MONT_CTX *mont)
+       {
+       BIGNUM *n;
+       BN_ULONG *ap,*np,*rp,n0,v,*nrp;
+       int al,nl,max,i,x,ri;
+
+       n= &(mont->N);
+       /* mont->ri is the size of mont->N in bits (rounded up
+          to the word size) */
+       al=ri=mont->ri/BN_BITS2;
+
+       nl=n->top;
+       if ((al == 0) || (nl == 0)) { ret->top=0; return(1); }
+
+       max=(nl+al+1); /* allow for overflow (no?) XXX */
+       if (bn_wexpand(r,max) == NULL) return(0);
+
+       r->neg^=n->neg;
+       np=n->d;
+       rp=r->d;
+       nrp= &(r->d[nl]);
+
+       /* clear the top words of T */
+       for (i=r->top; i<max; i++) /* memset? XXX */
+               r->d[i]=0;
+
+       r->top=max;
+       n0=mont->n0[0];
+
+#ifdef BN_COUNT
+       fprintf(stderr,"word BN_from_montgomery_word %d * %d\n",nl,nl);
+#endif
+       for (i=0; i<nl; i++)
+               {
+#ifdef __TANDEM
+                {
+                   long long t1;
+                   long long t2;
+                   long long t3;
+                   t1 = rp[0] * (n0 & 0177777);
+                   t2 = 037777600000l;
+                   t2 = n0 & t2;
+                   t3 = rp[0] & 0177777;
+                   t2 = (t3 * t2) & BN_MASK2;
+                   t1 = t1 + t2;
+                   v=bn_mul_add_words(rp,np,nl,(BN_ULONG) t1);
+                }
+#else
+               v=bn_mul_add_words(rp,np,nl,(rp[0]*n0)&BN_MASK2);
+#endif
+               nrp++;
+               rp++;
+               if (((nrp[-1]+=v)&BN_MASK2) >= v)
+                       continue;
+               else
+                       {
+                       if (((++nrp[0])&BN_MASK2) != 0) continue;
+                       if (((++nrp[1])&BN_MASK2) != 0) continue;
+                       for (x=2; (((++nrp[x])&BN_MASK2) == 0); x++) ;
+                       }
+               }
+       bn_correct_top(r);
+
+       /* mont->ri will be a multiple of the word size and below code
+        * is kind of BN_rshift(ret,r,mont->ri) equivalent */
+       if (r->top <= ri)
+               {
+               ret->top=0;
+               return(1);
+               }
+       al=r->top-ri;
+
+       if (bn_wexpand(ret,ri) == NULL) return(0);
+       x=0-(((al-ri)>>(sizeof(al)*8-1))&1);
+       ret->top=x=(ri&~x)|(al&x);      /* min(ri,al) */
+       ret->neg=r->neg;
+
+       rp=ret->d;
+       ap=&(r->d[ri]);
+
+       {
+       size_t m1,m2;
+
+       v=bn_sub_words(rp,ap,np,ri);
+       /* this ----------------^^ works even in al<ri case
+        * thanks to zealous zeroing of top of the vector in the
+        * beginning. */
+
+       /* if (al==ri && !v) || al>ri) nrp=rp; else nrp=ap; */
+       /* in other words if subtraction result is real, then
+        * trick unconditional memcpy below to perform in-place
+        * "refresh" instead of actual copy. */
+       m1=0-(size_t)(((al-ri)>>(sizeof(al)*8-1))&1);   /* al<ri */
+       m2=0-(size_t)(((ri-al)>>(sizeof(al)*8-1))&1);   /* al>ri */
+       m1|=m2;                 /* (al!=ri) */
+       m1|=(0-(size_t)v);      /* (al!=ri || v) */
+       m1&=~m2;                /* (al!=ri || v) && !al>ri */
+       nrp=(BN_ULONG *)(((size_t)rp&~m1)|((size_t)ap&m1));
+       }
+
+       /* 'i<ri' is chosen to eliminate dependency on input data, even
+        * though it results in redundant copy in al<ri case. */
+       for (i=0,ri-=4; i<ri; i+=4)
+               {
+               BN_ULONG t1,t2,t3,t4;
+               
+               t1=nrp[i+0];
+               t2=nrp[i+1];
+               t3=nrp[i+2];    ap[i+0]=0;
+               t4=nrp[i+3];    ap[i+1]=0;
+               rp[i+0]=t1;     ap[i+2]=0;
+               rp[i+1]=t2;     ap[i+3]=0;
+               rp[i+2]=t3;
+               rp[i+3]=t4;
+               }
+       for (ri+=4; i<ri; i++)
+               rp[i]=nrp[i], ap[i]=0;
+       bn_correct_top(r);
+       bn_correct_top(ret);
+       bn_check_top(ret);
+
+       return(1);
+       }
+
+int BN_from_montgomery(BIGNUM *ret, const BIGNUM *a, BN_MONT_CTX *mont,
+            BN_CTX *ctx)
+       {
+       int retn=0;
+       BIGNUM *t;
+
+       BN_CTX_start(ctx);
+       if ((t = BN_CTX_get(ctx)) && BN_copy(t,a))
+               retn = BN_from_montgomery_word(ret,t,mont);
+       BN_CTX_end(ctx);
+       return retn;
+       }
+#else
+
 int BN_from_montgomery(BIGNUM *ret, const BIGNUM *a, BN_MONT_CTX *mont,
             BN_CTX *ctx)
        {
@@ -357,6 +510,7 @@ int BN_from_montgomery(BIGNUM *ret, const BIGNUM *a, BN_MONT_CTX *mont,
        BN_CTX_end(ctx);
        return(retn);
        }
+#endif /* defined(MONT_WORD) && defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32) */
 
 BN_MONT_CTX *BN_MONT_CTX_new(void)
        {
@@ -376,6 +530,11 @@ void BN_MONT_CTX_init(BN_MONT_CTX *ctx)
        BN_init(&(ctx->RR));
        BN_init(&(ctx->N));
        BN_init(&(ctx->Ni));
+#if defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32) /* non-default compile */
+       ctx->n0[0] = ctx->n0[1] = 0;
+#else
+       ctx->n0 = 0;
+#endif
        ctx->flags=0;
        }
 
@@ -409,7 +568,11 @@ int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
 
                mont->ri=(BN_num_bits(mod)+(BN_BITS2-1))/BN_BITS2*BN_BITS2;
                BN_zero(R);
+#if defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32) /* non-default compile */
+               if (!(BN_set_bit(R,2*BN_BITS2))) goto err;      /* R */
+#else
                if (!(BN_set_bit(R,BN_BITS2))) goto err;        /* R */
+#endif
 
                buf[0]=mod->d[0]; /* tmod = N mod word size */
                buf[1]=0;
@@ -419,6 +582,35 @@ int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
                tmod.top = buf[0] != 0 ? 1 : 0;
                tmod.dmax=2;
                tmod.neg=0;
+
+#if defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32)
+                                                               tmod.top=0;
+               if ((buf[0] = mod->d[0]))                       tmod.top=1;
+               if ((buf[1] = mod->top>1 ? mod->d[1] : 0))      tmod.top=2;
+
+               if ((BN_mod_inverse(Ri,R,&tmod,ctx)) == NULL)
+                       goto err;
+               if (!BN_lshift(Ri,Ri,2*BN_BITS2)) goto err; /* R*Ri */
+               if (!BN_is_zero(Ri))
+                       {
+                       if (!BN_sub_word(Ri,1)) goto err;
+                       }
+               else /* if N mod word size == 1 */
+                       {
+                       if (bn_expand(Ri,(int)sizeof(BN_ULONG)*2) == NULL)
+                               goto err;
+                       /* Ri-- (mod double word size) */
+                       Ri->neg=0;
+                       Ri->d[0]=BN_MASK2;
+                       Ri->d[1]=BN_MASK2;
+                       Ri->top=2;
+                       }
+               if (!BN_div(Ri,NULL,Ri,&tmod,ctx)) goto err;
+               /* Ni = (R*Ri-1)/N,
+                * keep only couple of least significant words: */
+               mont->n0[0] = (Ri->top > 0) ? Ri->d[0] : 0;
+               mont->n0[1] = (Ri->top > 1) ? Ri->d[1] : 0;
+#else
                                                        /* Ri = R^-1 mod N*/
                if ((BN_mod_inverse(Ri,R,&tmod,ctx)) == NULL)
                        goto err;
@@ -435,6 +627,7 @@ int BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
                /* Ni = (R*Ri-1)/N,
                 * keep only least significant word: */
                mont->n0 = (Ri->top > 0) ? Ri->d[0] : 0;
+#endif
                }
 #else /* !MONT_WORD */
                { /* bignum version */
@@ -470,7 +663,12 @@ BN_MONT_CTX *BN_MONT_CTX_copy(BN_MONT_CTX *to, BN_MONT_CTX *from)
        if (!BN_copy(&(to->N),&(from->N))) return NULL;
        if (!BN_copy(&(to->Ni),&(from->Ni))) return NULL;
        to->ri=from->ri;
+#if defined(OPENSSL_BN_ASM_MONT) && (BN_BITS2<=32) /* non-default compile */
+       to->n0[0]=from->n0[0];
+       to->n0[1]=from->n0[1];
+#else
        to->n0=from->n0;
+#endif
        return(to);
        }