projects / openssl.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
crypto/poly1305: don't break carry chains.
[openssl.git] / crypto / poly1305 / asm / poly1305-x86.pl
diff --git a/crypto/poly1305/asm/poly1305-x86.pl b/crypto/poly1305/asm/poly1305-x86.pl
index 7c1aee5fe0b87325bf3c332a4754e1d2306ef823..97d0a81bea4377889cbf01dfe61ae8762a76960d 100755 (executable)
--- a/crypto/poly1305/asm/poly1305-x86.pl
+++ b/crypto/poly1305/asm/poly1305-x86.pl
@@ -38,6 +38,9 @@ $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
 push(@INC,"${dir}","${dir}../../perlasm");
 require "x86asm.pl";
 
+$output=pop;
+open STDOUT,">$output";
+
 &asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");
 
 $sse2=$avx=0;
@@ -296,6 +299,7 @@ if ($sse2) {
        &adc    ("ebx",0);
        &adc    ("ecx",0);
        &adc    ("esi",0);
+       &adc    ("edi",0);
 
        &cmp    ("ebp",&wparam(2));             # done yet?
        &jne    (&label("loop"));
@@ -536,6 +540,9 @@ my $base = shift; $base = "esp" if (!defined($base));
                             },"edx");
 
 sub lazy_reduction {
+my $extra = shift;
+my $paddx = defined($extra) ? paddq : paddd;
+
        ################################################################
        # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
        # and P. Schwabe
@@ -543,6 +550,7 @@ sub lazy_reduction {
         &movdqa        ($T0,$D3);
         &pand          ($D3,$MASK);
         &psrlq         ($T0,26);
+        &$extra        ()                              if (defined($extra));
         &paddq         ($T0,$D4);                      # h3 -> h4
        &movdqa         ($T1,$D0);
        &pand           ($D0,$MASK);
@@ -557,12 +565,12 @@ sub lazy_reduction {
                                                        # possible, because
                                                        # paddq is "broken"
                                                        # on Atom
-       &pand           ($D1,$MASK);
-       &paddq          ($T1,$D2);                      # h1 -> h2
         &psllq         ($T0,2);
+       &paddq          ($T1,$D2);                      # h1 -> h2
+        &$paddx        ($T0,$D0);                      # h4 -> h0
+       &pand           ($D1,$MASK);
        &movdqa         ($D2,$T1);
        &psrlq          ($T1,26);
-        &paddd         ($T0,$D0);                      # h4 -> h0
        &pand           ($D2,$MASK);
        &paddd          ($T1,$D3);                      # h2 -> h3
         &movdqa        ($D0,$T0);
@@ -1091,21 +1099,21 @@ my $addr = shift;
 
 &set_label("short_tail");
 
-       &lazy_reduction ();
-
        ################################################################
        # horizontal addition
 
+       &pshufd         ($T1,$D4,0b01001110);
+       &pshufd         ($T0,$D3,0b01001110);
+       &paddq          ($D4,$T1);
+       &paddq          ($D3,$T0);
        &pshufd         ($T1,$D0,0b01001110);
        &pshufd         ($T0,$D1,0b01001110);
-       &paddd          ($D0,$T1);
+       &paddq          ($D0,$T1);
+       &paddq          ($D1,$T0);
        &pshufd         ($T1,$D2,0b01001110);
-       &paddd          ($D1,$T0);
-       &pshufd         ($T0,$D3,0b01001110);
-       &paddd          ($D2,$T1);
-       &pshufd         ($T1,$D4,0b01001110);
-       &paddd          ($D3,$T0);
-       &paddd          ($D4,$T1);
+       #&paddq         ($D2,$T1);
+
+       &lazy_reduction (sub { &paddq ($D2,$T1) });
 
 &set_label("done");
        &movd           (&DWP(-16*3+4*0,"edi"),$D0);    # store hash value
@@ -1113,8 +1121,8 @@ my $addr = shift;
        &movd           (&DWP(-16*3+4*2,"edi"),$D2);
        &movd           (&DWP(-16*3+4*3,"edi"),$D3);
        &movd           (&DWP(-16*3+4*4,"edi"),$D4);
-&set_label("nodata");
        &mov    ("esp","ebp");
+&set_label("nodata");
 &function_end("_poly1305_blocks_sse2");
 
 &align (32);
@@ -1159,11 +1167,12 @@ my $addr = shift;
        &shr    ("edi",2);
        &lea    ("ebp",&DWP(0,"edi","edi",4));  # *5
         &mov   ("edi",&wparam(1));             # output
-       add     ("eax","ebp");
+       &add    ("eax","ebp");
         &mov   ("ebp",&wparam(2));             # key
-       adc     ("ebx",0);
-       adc     ("ecx",0);
-       adc     ("edx",0);
+       &adc    ("ebx",0);
+       &adc    ("ecx",0);
+       &adc    ("edx",0);
+       &adc    ("esi",0);
 
        &movd   ($D0,"eax");                    # offload original hash value
        &add    ("eax",5);                      # compare to modulus
@@ -1435,7 +1444,7 @@ sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
        &test   ("eax","eax");                          # is_base2_26?
        &jz     (&label("enter_blocks"));
 
-&set_label("enter_avx2",16);
+&set_label("enter_avx2");
        &vzeroupper     ();
 
        &call   (&label("pic_point"));
@@ -1702,18 +1711,18 @@ sub vlazy_reduction {
        &vpsrlq         ($T1,$D1,26);
        &vpand          ($D1,$D1,$MASK);
        &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
-        &vpaddd        ($D0,$D0,$T0);
+        &vpaddq        ($D0,$D0,$T0);
         &vpsllq        ($T0,$T0,2);
        &vpsrlq         ($T1,$D2,26);
        &vpand          ($D2,$D2,$MASK);
-        &vpaddd        ($D0,$D0,$T0);                  # h4 -> h0
-       &vpaddd         ($D3,$D3,$T1);                  # h2 -> h3
+        &vpaddq        ($D0,$D0,$T0);                  # h4 -> h0
+       &vpaddq         ($D3,$D3,$T1);                  # h2 -> h3
        &vpsrlq         ($T1,$D3,26);
         &vpsrlq        ($T0,$D0,26);
         &vpand         ($D0,$D0,$MASK);
        &vpand          ($D3,$D3,$MASK);
-        &vpaddd        ($D1,$D1,$T0);                  # h0 -> h1
-       &vpaddd         ($D4,$D4,$T1);                  # h3 -> h4
+        &vpaddq        ($D1,$D1,$T0);                  # h0 -> h1
+       &vpaddq         ($D4,$D4,$T1);                  # h3 -> h4
 }
        &vlazy_reduction();
 
@@ -1731,31 +1740,31 @@ sub vlazy_reduction {
 
        &vpmuladd       (sub {  my $i=shift; &QWP(4+32*$i-128,"edx");   });
 
-       &vlazy_reduction();
-
        ################################################################
        # horizontal addition
 
+       &vpsrldq        ($T0,$D4,8);
+       &vpsrldq        ($T1,$D3,8);
+       &vpaddq         ($D4,$D4,$T0);
        &vpsrldq        ($T0,$D0,8);
+       &vpaddq         ($D3,$D3,$T1);
        &vpsrldq        ($T1,$D1,8);
        &vpaddq         ($D0,$D0,$T0);
        &vpsrldq        ($T0,$D2,8);
        &vpaddq         ($D1,$D1,$T1);
-       &vpsrldq        ($T1,$D3,8);
+       &vpermq         ($T1,$D4,2);                    # keep folding
        &vpaddq         ($D2,$D2,$T0);
-       &vpsrldq        ($T0,$D4,8);
-       &vpaddq         ($D3,$D3,$T1);
-       &vpermq         ($T1,$D0,2);                    # keep folding
-       &vpaddq         ($D4,$D4,$T0);
+       &vpermq         ($T0,$D3,2);
+       &vpaddq         ($D4,$D4,$T1);
+       &vpermq         ($T1,$D0,2);
+       &vpaddq         ($D3,$D3,$T0);
        &vpermq         ($T0,$D1,2);
        &vpaddq         ($D0,$D0,$T1);
        &vpermq         ($T1,$D2,2);
        &vpaddq         ($D1,$D1,$T0);
-       &vpermq         ($T0,$D3,2);
        &vpaddq         ($D2,$D2,$T1);
-       &vpermq         ($T1,$D4,2);
-       &vpaddq         ($D3,$D3,$T0);
-       &vpaddq         ($D4,$D4,$T1);
+
+       &vlazy_reduction();
 
        &cmp            ("ecx",0);
        &je             (&label("done"));
@@ -1772,14 +1781,14 @@ sub vlazy_reduction {
        &jmp            (&label("even"));
 
 &set_label("done",16);
-       &vmovd          (&DWP(-16*3+4*0,"edi"),"xmm0"); # store hash value
-       &vmovd          (&DWP(-16*3+4*1,"edi"),"xmm1");
-       &vmovd          (&DWP(-16*3+4*2,"edi"),"xmm2");
-       &vmovd          (&DWP(-16*3+4*3,"edi"),"xmm3");
-       &vmovd          (&DWP(-16*3+4*4,"edi"),"xmm4");
+       &vmovd          (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
+       &vmovd          (&DWP(-16*3+4*1,"edi"),&X($D1));
+       &vmovd          (&DWP(-16*3+4*2,"edi"),&X($D2));
+       &vmovd          (&DWP(-16*3+4*3,"edi"),&X($D3));
+       &vmovd          (&DWP(-16*3+4*4,"edi"),&X($D4));
        &vzeroupper     ();
-&set_label("nodata");
        &mov    ("esp","ebp");
+&set_label("nodata");
 &function_end("_poly1305_blocks_avx2");
 }
 &set_label("const_sse2",64);
@@ -1792,3 +1801,5 @@ sub vlazy_reduction {
 &align (4);
 
 &asm_finish();
+
+close STDOUT;
Unnamed repository; edit this file 'description' to name the repository.