SSE2 accelerated bn_mul_add_words. Code is currently disabled till proper

[openssl.git] / crypto / bn / asm / bn-586.pl

diff --git a/crypto/bn/asm/bn-586.pl b/crypto/bn/asm/bn-586.pl
index 7a03c67b5b1fec3852d3b5c1b9e817963a216ed0..9a7c64c46455dad6cc0754107f456ccb9f886e5b 100644 (file)
--- a/crypto/bn/asm/bn-586.pl
+++ b/crypto/bn/asm/bn-586.pl
@@ -3,7 +3,9 @@
 push(@INC,"perlasm","../../perlasm");
 require "x86asm.pl";
 
 push(@INC,"perlasm","../../perlasm");
 require "x86asm.pl";
 

-&asm_init($ARGV[0],"bn-586.pl");
+&asm_init($ARGV[0],$0);
+
+$sse2=0;       # SSE2 is temporarily disabled...

 
 &bn_mul_add_words("bn_mul_add_words");
 &bn_mul_words("bn_mul_words");
 
 &bn_mul_add_words("bn_mul_add_words");
 &bn_mul_words("bn_mul_words");


@@ -11,6 +13,7 @@ require "x86asm.pl";
 &bn_div_words("bn_div_words");
 &bn_add_words("bn_add_words");
 &bn_sub_words("bn_sub_words");
 &bn_div_words("bn_div_words");
 &bn_add_words("bn_add_words");
 &bn_sub_words("bn_sub_words");

+&bn_sub_part_words("bn_sub_part_words");

 
 &asm_finish();
 
 
 &asm_finish();
 


@@ -41,6 +44,83 @@ sub bn_mul_add_words
 
        &jz(&label("maw_finish"));
 
 
        &jz(&label("maw_finish"));
 

+       if ($sse2) {
+               &picmeup("eax","OPENSSL_ia32cap");
+               &bt(&DWP(0,"eax"),26);
+               &jnc(&label("maw_loop"));
+
+               &movd("mm0",$w);                # mm0 = w
+               &pxor("mm1","mm1");             # mm1 = carry_in
+
+               &set_label("maw_sse2_loop",0);
+               &movd("mm3",&DWP(0,$r,"",0));   # mm3 = r[0]
+               &paddq("mm1","mm3");            # mm1 = carry_in + r[0]
+               &movd("mm2",&DWP(0,$a,"",0));   # mm2 = a[0]
+               &pmuludq("mm2","mm0");          # mm2 = w*a[0]
+               &movd("mm4",&DWP(4,$a,"",0));   # mm4 = a[1]
+               &pmuludq("mm4","mm0");          # mm4 = w*a[1]
+               &movd("mm6",&DWP(8,$a,"",0));   # mm6 = a[2]
+               &pmuludq("mm6","mm0");          # mm6 = w*a[2]
+               &movd("mm7",&DWP(12,$a,"",0));  # mm7 = a[3]
+               &pmuludq("mm7","mm0");          # mm7 = w*a[3]
+               &paddq("mm1","mm2");            # mm1 = carry_in + r[0] + w*a[0]
+               &movd("mm3",&DWP(4,$r,"",0));   # mm3 = r[1]
+               &paddq("mm3","mm4");            # mm3 = r[1] + w*a[1]
+               &movd("mm5",&DWP(8,$r,"",0));   # mm5 = r[2]
+               &paddq("mm5","mm6");            # mm5 = r[2] + w*a[2]
+               &movd("mm4",&DWP(12,$r,"",0));  # mm4 = r[3]
+               &paddq("mm7","mm4");            # mm7 = r[3] + w*a[3]
+               &movd(&DWP(0,$r,"",0),"mm1");
+               &movd("mm2",&DWP(16,$a,"",0));  # mm2 = a[4]
+               &pmuludq("mm2","mm0");          # mm2 = w*a[4]
+               &psrlq("mm1",32);               # mm1 = carry0
+               &movd("mm4",&DWP(20,$a,"",0));  # mm4 = a[5]
+               &pmuludq("mm4","mm0");          # mm4 = w*a[5]
+               &paddq("mm1","mm3");            # mm1 = carry0 + r[1] + w*a[1]
+               &movd("mm6",&DWP(24,$a,"",0));  # mm6 = a[6]
+               &pmuludq("mm6","mm0");          # mm6 = w*a[6]
+               &movd(&DWP(4,$r,"",0),"mm1");
+               &psrlq("mm1",32);               # mm1 = carry1
+               &movd("mm3",&DWP(28,$a,"",0));  # mm3 = a[7]
+               &add($a,32);
+               &pmuludq("mm3","mm0");          # mm3 = w*a[7]
+               &paddq("mm1","mm5");            # mm1 = carry1 + r[2] + w*a[2]
+               &movd("mm5",&DWP(16,$r,"",0));  # mm5 = r[4]
+               &paddq("mm2","mm5");            # mm2 = r[4] + w*a[4]
+               &movd(&DWP(8,$r,"",0),"mm1");
+               &psrlq("mm1",32);               # mm1 = carry2
+               &paddq("mm1","mm7");            # mm1 = carry2 + r[3] + w*a[3]
+               &movd("mm5",&DWP(20,$r,"",0));  # mm5 = r[5]
+               &paddq("mm4","mm5");            # mm4 = r[5] + w*a[5]
+               &movd(&DWP(12,$r,"",0),"mm1");
+               &psrlq("mm1",32);               # mm1 = carry3
+               &paddq("mm1","mm2");            # mm1 = carry3 + r[4] + w*a[4]
+               &movd("mm5",&DWP(24,$r,"",0));  # mm5 = r[6]
+               &paddq("mm6","mm5");            # mm6 = r[6] + w*a[6]
+               &movd(&DWP(16,$r,"",0),"mm1");
+               &psrlq("mm1",32);               # mm1 = carry4
+               &paddq("mm1","mm4");            # mm1 = carry4 + r[5] + w*a[5]
+               &movd("mm5",&DWP(28,$r,"",0));  # mm5 = r[7]
+               &paddq("mm3","mm5");            # mm3 = r[7] + w*a[7]
+               &movd(&DWP(20,$r,"",0),"mm1");
+               &psrlq("mm1",32);               # mm1 = carry5
+               &paddq("mm1","mm6");            # mm1 = carry5 + r[6] + w*a[6]
+               &movd(&DWP(24,$r,"",0),"mm1");
+               &psrlq("mm1",32);               # mm1 = carry6
+               &paddq("mm1","mm3");            # mm1 = carry6 + r[7] + w*a[7]
+               &movd(&DWP(28,$r,"",0),"mm1");
+               &add($r,32);
+               &psrlq("mm1",32);               # mm1 = carry_out
+
+               &sub("ecx",8);
+               &jnz(&label("maw_sse2_loop"));
+
+               &movd($c,"mm1");                # c = carry_out
+               &emms();
+
+               &jmp(&label("maw_finish"));
+       }
+

        &set_label("maw_loop",0);
 
        &mov(&swtmp(0),"ecx");  #
        &set_label("maw_loop",0);
 
        &mov(&swtmp(0),"ecx");  #


@@ -300,7 +380,7 @@ sub bn_add_words
                 &add($tmp1,$tmp2);
                &adc($c,0);
                 &dec($num) if ($i != 6);
                 &add($tmp1,$tmp2);
                &adc($c,0);
                 &dec($num) if ($i != 6);

-               &mov(&DWP($i*4,$r,"",0),$tmp1); # *a
+               &mov(&DWP($i*4,$r,"",0),$tmp1); # *r

                 &jz(&label("aw_end")) if ($i != 6);
                }
        &set_label("aw_end",0);
                 &jz(&label("aw_end")) if ($i != 6);
                }
        &set_label("aw_end",0);


@@ -372,11 +452,219 @@ sub bn_sub_words
                 &sub($tmp1,$tmp2);
                &adc($c,0);
                 &dec($num) if ($i != 6);
                 &sub($tmp1,$tmp2);
                &adc($c,0);
                 &dec($num) if ($i != 6);

-               &mov(&DWP($i*4,$r,"",0),$tmp1); # *a
+               &mov(&DWP($i*4,$r,"",0),$tmp1); # *r
+                &jz(&label("aw_end")) if ($i != 6);
+               }
+       &set_label("aw_end",0);
+
+#      &mov("eax",$c);         # $c is "eax"
+
+       &function_end($name);
+       }
+
+sub bn_sub_part_words
+       {
+       local($name)=@_;
+
+       &function_begin($name,"");
+
+       &comment("");
+       $a="esi";
+       $b="edi";
+       $c="eax";
+       $r="ebx";
+       $tmp1="ecx";
+       $tmp2="edx";
+       $num="ebp";
+
+       &mov($r,&wparam(0));    # get r
+        &mov($a,&wparam(1));   # get a
+       &mov($b,&wparam(2));    # get b
+        &mov($num,&wparam(3)); # get num
+       &xor($c,$c);            # clear carry
+        &and($num,0xfffffff8); # num / 8
+
+       &jz(&label("aw_finish"));
+
+       &set_label("aw_loop",0);
+       for ($i=0; $i<8; $i++)
+               {
+               &comment("Round $i");
+
+               &mov($tmp1,&DWP($i*4,$a,"",0));         # *a
+                &mov($tmp2,&DWP($i*4,$b,"",0));        # *b
+               &sub($tmp1,$c);
+                &mov($c,0);
+               &adc($c,$c);
+                &sub($tmp1,$tmp2);
+               &adc($c,0);
+                &mov(&DWP($i*4,$r,"",0),$tmp1);        # *r
+               }
+
+       &comment("");
+       &add($a,32);
+        &add($b,32);
+       &add($r,32);
+        &sub($num,8);
+       &jnz(&label("aw_loop"));
+
+       &set_label("aw_finish",0);
+       &mov($num,&wparam(3));  # get num
+       &and($num,7);
+        &jz(&label("aw_end"));
+
+       for ($i=0; $i<7; $i++)
+               {
+               &comment("Tail Round $i");
+               &mov($tmp1,&DWP(0,$a,"",0));    # *a
+                &mov($tmp2,&DWP(0,$b,"",0));# *b
+               &sub($tmp1,$c);
+                &mov($c,0);
+               &adc($c,$c);
+                &sub($tmp1,$tmp2);
+               &adc($c,0);
+               &mov(&DWP(0,$r,"",0),$tmp1);    # *r
+               &add($a, 4);
+               &add($b, 4);
+               &add($r, 4);
+                &dec($num) if ($i != 6);

                 &jz(&label("aw_end")) if ($i != 6);
                }
        &set_label("aw_end",0);
 
                 &jz(&label("aw_end")) if ($i != 6);
                }
        &set_label("aw_end",0);
 

+       &cmp(&wparam(4),0);
+       &je(&label("pw_end"));
+
+       &mov($num,&wparam(4));  # get dl
+       &cmp($num,0);
+       &je(&label("pw_end"));
+       &jge(&label("pw_pos"));
+
+       &comment("pw_neg");
+       &mov($tmp2,0);
+       &sub($tmp2,$num);
+       &mov($num,$tmp2);
+       &and($num,0xfffffff8);  # num / 8
+       &jz(&label("pw_neg_finish"));
+
+       &set_label("pw_neg_loop",0);
+       for ($i=0; $i<8; $i++)
+       {
+           &comment("dl<0 Round $i");
+
+           &mov($tmp1,0);
+           &mov($tmp2,&DWP($i*4,$b,"",0));     # *b
+           &sub($tmp1,$c);
+           &mov($c,0);
+           &adc($c,$c);
+           &sub($tmp1,$tmp2);
+           &adc($c,0);
+           &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
+       }
+           
+       &comment("");
+       &add($b,32);
+       &add($r,32);
+       &sub($num,8);
+       &jnz(&label("pw_neg_loop"));
+           
+       &set_label("pw_neg_finish",0);
+       &mov($tmp2,&wparam(4)); # get dl
+       &mov($num,0);
+       &sub($num,$tmp2);
+       &and($num,7);
+       &jz(&label("pw_end"));
+           
+       for ($i=0; $i<7; $i++)
+       {
+           &comment("dl<0 Tail Round $i");
+           &mov($tmp1,0);
+           &mov($tmp2,&DWP($i*4,$b,"",0));# *b
+           &sub($tmp1,$c);
+           &mov($c,0);
+           &adc($c,$c);
+           &sub($tmp1,$tmp2);
+           &adc($c,0);
+           &dec($num) if ($i != 6);
+           &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
+           &jz(&label("pw_end")) if ($i != 6);
+       }
+
+       &jmp(&label("pw_end"));
+       
+       &set_label("pw_pos",0);
+       
+       &and($num,0xfffffff8);  # num / 8
+       &jz(&label("pw_pos_finish"));
+
+       &set_label("pw_pos_loop",0);
+
+       for ($i=0; $i<8; $i++)
+       {
+           &comment("dl>0 Round $i");
+
+           &mov($tmp1,&DWP($i*4,$a,"",0));     # *a
+           &sub($tmp1,$c);
+           &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
+           &jnc(&label("pw_nc".$i));
+       }
+           
+       &comment("");
+       &add($a,32);
+       &add($r,32);
+       &sub($num,8);
+       &jnz(&label("pw_pos_loop"));
+           
+       &set_label("pw_pos_finish",0);
+       &mov($num,&wparam(4));  # get dl
+       &and($num,7);
+       &jz(&label("pw_end"));
+           
+       for ($i=0; $i<7; $i++)
+       {
+           &comment("dl>0 Tail Round $i");
+           &mov($tmp1,&DWP($i*4,$a,"",0));     # *a
+           &sub($tmp1,$c);
+           &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
+           &jnc(&label("pw_tail_nc".$i));
+           &dec($num) if ($i != 6);
+           &jz(&label("pw_end")) if ($i != 6);
+       }
+       &mov($c,1);
+       &jmp(&label("pw_end"));
+
+       &set_label("pw_nc_loop",0);
+       for ($i=0; $i<8; $i++)
+       {
+           &mov($tmp1,&DWP($i*4,$a,"",0));     # *a
+           &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
+           &set_label("pw_nc".$i,0);
+       }
+           
+       &comment("");
+       &add($a,32);
+       &add($r,32);
+       &sub($num,8);
+       &jnz(&label("pw_nc_loop"));
+           
+       &mov($num,&wparam(4));  # get dl
+       &and($num,7);
+       &jz(&label("pw_nc_end"));
+           
+       for ($i=0; $i<7; $i++)
+       {
+           &mov($tmp1,&DWP($i*4,$a,"",0));     # *a
+           &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
+           &set_label("pw_tail_nc".$i,0);
+           &dec($num) if ($i != 6);
+           &jz(&label("pw_nc_end")) if ($i != 6);
+       }
+
+       &set_label("pw_nc_end",0);
+       &mov($c,0);
+
+       &set_label("pw_end",0);
+

 #      &mov("eax",$c);         # $c is "eax"
 
        &function_end($name);
 #      &mov("eax",$c);         # $c is "eax"
 
        &function_end($name);