ghash-x86.pl: engage original MMX version in no-sse2 builds.

[openssl.git] / crypto / modes / asm / ghash-x86.pl

diff --git a/crypto/modes/asm/ghash-x86.pl b/crypto/modes/asm/ghash-x86.pl
index fca19e41f0849da914eb65f356e34ab3811ff320..6b09669d474abbc55c8643b4460f0b6de770eab6 100644 (file)
--- a/crypto/modes/asm/ghash-x86.pl
+++ b/crypto/modes/asm/ghash-x86.pl
@@ -103,6 +103,16 @@
 # providing access to a Westmere-based system on behalf of Intel
 # Open Source Technology Centre.
 
+# January 2010
+#
+# Tweaked to optimize transitions between integer and FP operations
+# on same XMM register, PCLMULQDQ subroutine was measured to process
+# one byte in 2.07 cycles on Sandy Bridge, and in 2.12 - on Westmere.
+# The minor regression on Westmere is outweighed by ~15% improvement
+# on Sandy Bridge. Strangely enough attempt to modify 64-bit code in
+# similar manner resulted in almost 20% degradation on Sandy Bridge,
+# where original 64-bit code processes one byte in 1.95 cycles.
+
 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
 push(@INC,"${dir}","${dir}../../perlasm");
 require "x86asm.pl";
@@ -321,7 +331,7 @@ if (!$x86only) {{{
 
 &static_label("rem_4bit");
 
-if (0) {{      # "May" MMX version is kept for reference...
+if (!$sse2) {{ # pure-MMX "May" version...
 
 $S=12;         # shift factor for rem_4bit
 
@@ -811,12 +821,6 @@ $len="ebx";
 
 &static_label("bswap");
 
-sub pclmulqdq
-{ my($dst,$src,$imm)=@_;
-    if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
-    {  &data_byte(0x66,0x0f,0x3a,0x44,0xc0|($1<<3)|$2,$imm);   }
-}
-
 sub clmul64x64_T2 {    # minimal "register" pressure
 my ($Xhi,$Xi,$Hkey)=@_;
 
@@ -829,8 +833,8 @@ my ($Xhi,$Xi,$Hkey)=@_;
        &pclmulqdq      ($Xi,$Hkey,0x00);       #######
        &pclmulqdq      ($Xhi,$Hkey,0x11);      #######
        &pclmulqdq      ($T1,$T2,0x00);         #######
-       &pxor           ($T1,$Xi);              #
-       &pxor           ($T1,$Xhi);             #
+       &xorps          ($T1,$Xi);              #
+       &xorps          ($T1,$Xhi);             #
 
        &movdqa         ($T2,$T1);              #
        &psrldq         ($T1,8);
@@ -950,7 +954,7 @@ my ($Xhi,$Xi) = @_;
 
        &movdqu         ($Xi,&QWP(0,$Xip));
        &movdqa         ($T3,&QWP(0,$const));
-       &movdqu         ($Hkey,&QWP(0,$Htbl));
+       &movups         ($Hkey,&QWP(0,$Htbl));
        &pshufb         ($Xi,$T3);
 
        &clmul64x64_T2  ($Xhi,$Xi,$Hkey);
@@ -993,7 +997,7 @@ my ($Xhi,$Xi) = @_;
        &pxor           ($Xi,$T1);              # Ii+Xi
 
        &clmul64x64_T2  ($Xhn,$Xn,$Hkey);       # H*Ii+1
-       &movdqu         ($Hkey,&QWP(16,$Htbl)); # load H^2
+       &movups         ($Hkey,&QWP(16,$Htbl)); # load H^2
 
        &lea            ($inp,&DWP(32,$inp));   # i+=2
        &sub            ($len,0x20);
@@ -1002,7 +1006,7 @@ my ($Xhi,$Xi) = @_;
 &set_label("mod_loop");
        &clmul64x64_T2  ($Xhi,$Xi,$Hkey);       # H^2*(Ii+Xi)
        &movdqu         ($T1,&QWP(0,$inp));     # Ii
-       &movdqu         ($Hkey,&QWP(0,$Htbl));  # load H
+       &movups         ($Hkey,&QWP(0,$Htbl));  # load H
 
        &pxor           ($Xi,$Xn);              # (H*Ii+1) + H^2*(Ii+Xi)
        &pxor           ($Xhi,$Xhn);
@@ -1043,9 +1047,9 @@ my ($Xhi,$Xi) = @_;
          &pxor         ($Xi,$T2);              #
 
        &pclmulqdq      ($T1,$T3,0x00);         #######
-       &movdqu         ($Hkey,&QWP(16,$Htbl)); # load H^2
-       &pxor           ($T1,$Xn);              #
-       &pxor           ($T1,$Xhn);             #
+       &movups         ($Hkey,&QWP(16,$Htbl)); # load H^2
+       &xorps          ($T1,$Xn);              #
+       &xorps          ($T1,$Xhn);             #
 
        &movdqa         ($T3,$T1);              #
        &psrldq         ($T1,8);
@@ -1069,7 +1073,7 @@ my ($Xhi,$Xi) = @_;
        &test           ($len,$len);
        &jnz            (&label("done"));
 
-       &movdqu         ($Hkey,&QWP(0,$Htbl));  # load H
+       &movups         ($Hkey,&QWP(0,$Htbl));  # load H
 &set_label("odd_tail");
        &movdqu         ($T1,&QWP(0,$inp));     # Ii
        &pshufb         ($T1,$T3);