From: Andy Polyakov Date: Wed, 24 Oct 2012 08:21:10 +0000 (+0000) Subject: ghash-sparcv9.pl: add VIS3 code path. X-Git-Tag: master-post-reformat~1653 X-Git-Url: https://git.openssl.org/?p=openssl.git;a=commitdiff_plain;h=23328d4b275e0d4bdb71697d3f95c1100380de5e ghash-sparcv9.pl: add VIS3 code path. --- diff --git a/crypto/modes/asm/ghash-sparcv9.pl b/crypto/modes/asm/ghash-sparcv9.pl index 70e7b044a3..d8f7272c58 100644 --- a/crypto/modes/asm/ghash-sparcv9.pl +++ b/crypto/modes/asm/ghash-sparcv9.pl @@ -36,6 +36,13 @@ # references to input data and Z.hi updates to achieve 12 cycles # timing. To anchor to something else, sha1-sparcv9.pl spends 11.6 # cycles to process one byte on UltraSPARC pre-Tx CPU and ~24 on T1. +# +# October 2012 +# +# Add VIS3 lookup-table-free implementation using polynomial +# multiplication xmulx[hi] and extended addition addxc[cc] +# instructions. 3.96/6.26x improvement on T3/T4 or in absolute +# terms 9.02/2.61 cycles per byte. $bits=32; for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } @@ -66,6 +73,10 @@ $Htbl="%i1"; $inp="%i2"; $len="%i3"; +$code.=<<___ if ($bits==64); +.register %g2,#scratch +.register %g3,#scratch +___ $code.=<<___; .section ".text",#alloc,#execinstr @@ -321,10 +332,213 @@ gcm_gmult_4bit: restore .type gcm_gmult_4bit,#function .size gcm_gmult_4bit,(.-gcm_gmult_4bit) -.asciz "GHASH for SPARCv9, CRYPTOGAMS by " +___ + +{{{ +# Straightforward 64-bits-at-a-time approach with pair of 128x64-bit +# multiplications followed by 64-bit reductions. While it might be +# suboptimal with regard to sheer amount of multiplications, other +# methods would require larger amount of 64-bit registers, which we +# don't have in 32-bit application. Also, they [alternative methods +# such as aggregated reduction] kind of thrive on fast 128-bit SIMD +# instructions and these are not option on SPARC... + +($Xip,$Htable,$inp,$len)=map("%i$_",(0..3)); + +($xE1,$Hhi,$Hlo,$Rhi,$Rlo,$M0hi,$M0lo,$M1hi,$M1lo,$Zhi,$Zlo,$X)= + (map("%g$_",(1..5)),map("%o$_",(0..5,7))); +($shl,$shr)=map("%l$_",(0..7)); + +$code.=<<___; +.globl gcm_gmult_vis3 +.align 32 +gcm_gmult_vis3: + save %sp,-$frame,%sp + + ldx [$Xip+8],$X ! load X.lo + ldx [$Htable-8], $Hlo ! load H + ldx [$Htable-16],$Hhi + mov 0xE1,$xE1 + sllx $xE1,57,$xE1 + + xmulx $X,$Hlo,$M0lo ! H·X.lo + xmulxhi $X,$Hlo,$M0hi + xmulx $X,$Hhi,$M1lo + xmulxhi $X,$Hhi,$M1hi + ldx [$Xip+0],$X ! load X.hi + + addcc $M0lo,$M0lo,$M0lo ! (H·X.lo)<<1 + xor $M0hi,$M1lo,$M1lo + + xmulx $xE1,$M0lo,$Rlo ! res=Z.lo·(0xE1<<57) + xmulxhi $xE1,$M0lo,$Rhi + + addxccc $M1lo,$M1lo,$Zlo ! Z=((H·X.lo)<<1)>>64 + addxc $M1hi,$M1hi,$Zhi + xor $M0lo,$Zhi,$Zhi ! overflow bit from 0xE1<<57 + + xmulx $X,$Hlo,$M0lo ! H·X.hi + xmulxhi $X,$Hlo,$M0hi + xmulx $X,$Hhi,$M1lo + xmulxhi $X,$Hhi,$M1hi + + xor $Rlo,$Zlo,$Zlo ! Z^=res + xor $Rhi,$Zhi,$Zhi + + addcc $M0lo,$M0lo,$M0lo ! (H·X.lo)<<1 + xor $Zlo, $M0lo,$M0lo + xor $M0hi,$M1lo,$M1lo + + xmulx $xE1,$M0lo,$Rlo ! res=Z.lo·(0xE1<<57) + xmulxhi $xE1,$M0lo,$Rhi + + addxccc $M1lo,$M1lo,$M1lo + addxc $M1hi,$M1hi,$M1hi + + xor $M1lo,$Zhi,$Zlo ! Z=(Z^(H·X.hi)<<1)>>64 + xor $M0lo,$M1hi,$Zhi ! overflow bit from 0xE1<<57 + + xor $Rlo,$Zlo,$Zlo ! Z^=res + xor $Rhi,$Zhi,$Zhi + + stx $Zlo,[$Xip+8] ! save Xi + stx $Zhi,[$Xip+0] + + ret + restore +.type gcm_gmult_vis3,#function +.size gcm_gmult_vis3,.-gcm_gmult_vis3 + +.globl gcm_ghash_vis3 +.align 32 +gcm_ghash_vis3: + save %sp,-$frame,%sp + + ldx [$Xip+0],$Zhi ! load X.hi + ldx [$Xip+8],$Zlo ! load X.lo + and $inp,7,$shl + andn $inp,7,$inp + ldx [$Htable-8], $Hlo ! load H + ldx [$Htable-16],$Hhi + sll $shl,3,$shl + prefetch [$inp+63], 20 + mov 0xE1,$xE1 + sub %g0,$shl,$shr + sllx $xE1,57,$xE1 + +.Loop: + ldx [$inp+8],$Rlo ! load *inp + brz,pt $shl,1f + ldx [$inp+0],$Rhi + + ldx [$inp+16],$X ! align data + srlx $Rlo,$shr,$M0lo + sllx $Rlo,$shl,$Rlo + sllx $Rhi,$shl,$Rhi + srlx $X,$shr,$X + or $M0lo,$Rhi,$Rhi + or $X,$Rlo,$Rlo + +1: + add $inp,16,$inp + sub $len,16,$len + xor $Rlo,$Zlo,$X + prefetch [$inp+63], 20 + + xmulx $X,$Hlo,$M0lo ! H·X.lo + xmulxhi $X,$Hlo,$M0hi + xmulx $X,$Hhi,$M1lo + xmulxhi $X,$Hhi,$M1hi + xor $Rhi,$Zhi,$X + + addcc $M0lo,$M0lo,$M0lo ! (H·X.lo)<<1 + xor $M0hi,$M1lo,$M1lo + + xmulx $xE1,$M0lo,$Rlo ! res=Z.lo·(0xE1<<57) + xmulxhi $xE1,$M0lo,$Rhi + + addxccc $M1lo,$M1lo,$Zlo ! Z=((H·X.lo)<<1)>>64 + addxc $M1hi,$M1hi,$Zhi + xor $M0lo,$Zhi,$Zhi ! overflow bit from 0xE1<<57 + + xmulx $X,$Hlo,$M0lo ! H·X.hi + xmulxhi $X,$Hlo,$M0hi + xmulx $X,$Hhi,$M1lo + xmulxhi $X,$Hhi,$M1hi + + xor $Rlo,$Zlo,$Zlo ! Z^=res + xor $Rhi,$Zhi,$Zhi + + addcc $M0lo,$M0lo,$M0lo ! (H·X.lo)<<1 + xor $Zlo, $M0lo,$M0lo + xor $M0hi,$M1lo,$M1lo + + xmulx $xE1,$M0lo,$Rlo ! res=Z.lo·(0xE1<<57) + xmulxhi $xE1,$M0lo,$Rhi + + addxccc $M1lo,$M1lo,$M1lo + addxc $M1hi,$M1hi,$M1hi + + xor $M1lo,$Zhi,$Zlo ! Z=(Z^(H·X.hi)<<1)>>64 + xor $M0lo,$M1hi,$Zhi ! overflow bit from 0xE1<<57 + + xor $Rlo,$Zlo,$Zlo ! Z^=res + brnz,pt $len,.Loop + xor $Rhi,$Zhi,$Zhi + + stx $Zlo,[$Xip+8] ! save Xi + stx $Zhi,[$Xip+0] + + ret + restore +.type gcm_ghash_vis3,#function +.size gcm_ghash_vis3,.-gcm_ghash_vis3 +___ +}}} +$code.=<<___; +.asciz "GHASH for SPARCv9/VIS3, CRYPTOGAMS by " .align 4 ___ -$code =~ s/\`([^\`]*)\`/eval $1/gem; -print $code; + +# Purpose of these subroutines is to explicitly encode VIS instructions, +# so that one can compile the module without having to specify VIS +# extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. +# Idea is to reserve for option to produce "universal" binary and let +# programmer detect if current CPU is VIS capable at run-time. +sub unvis3 { +my ($mnemonic,$rs1,$rs2,$rd)=@_; +my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); +my ($ref,$opf); +my %visopf = ( "addxc" => 0x011, + "addxccc" => 0x013, + "xmulx" => 0x115, + "xmulxhi" => 0x116 ); + + $ref = "$mnemonic\t$rs1,$rs2,$rd"; + + if ($opf=$visopf{$mnemonic}) { + foreach ($rs1,$rs2,$rd) { + return $ref if (!/%([goli])([0-9])/); + $_=$bias{$1}+$2; + } + + return sprintf ".word\t0x%08x !%s", + 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/\b(xmulx[hi]*|addxc[c]{0,2})\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/ + &unvis3($1,$2,$3,$4) + /ge; + + print $_,"\n"; +} + close STDOUT; diff --git a/crypto/modes/gcm128.c b/crypto/modes/gcm128.c index f8dd497f87..0b8dc8fb32 100644 --- a/crypto/modes/gcm128.c +++ b/crypto/modes/gcm128.c @@ -674,6 +674,13 @@ void gcm_ghash_4bit_x86(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len void gcm_gmult_neon(u64 Xi[2],const u128 Htable[16]); void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); # endif +# elif defined(__sparc__) || defined(__sparc) +# include "sparc_arch.h" +# define GHASH_ASM_SPARC +# define GCM_FUNCREF_4BIT +extern unsigned int OPENSSL_sparcv9cap_P[]; +void gcm_gmult_vis3(u64 Xi[2],const u128 Htable[16]); +void gcm_ghash_vis3(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); # endif #endif @@ -750,6 +757,15 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block) ctx->gmult = gcm_gmult_4bit; ctx->ghash = gcm_ghash_4bit; } +# elif defined(GHASH_ASM_SPARC) + if (OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3) { + ctx->gmult = gcm_gmult_vis3; + ctx->ghash = gcm_ghash_vis3; + } else { + gcm_init_4bit(ctx->Htable,ctx->H.u); + ctx->gmult = gcm_gmult_4bit; + ctx->ghash = gcm_ghash_4bit; + } # else gcm_init_4bit(ctx->Htable,ctx->H.u); # endif