3 # ====================================================================
4 # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
12 # Provided that UltraSPARC VIS instructions are pipe-lined(*) and
13 # pairable(*) with IALU ones, offloading of Xupdate to the UltraSPARC
14 # Graphic Unit would make it possible to achieve higher instruction-
15 # level parallelism, ILP, and thus higher performance. It should be
16 # explicitly noted that ILP is the keyword, and it means that this
17 # code would be unsuitable for cores like UltraSPARC-Tx. The idea is
18 # not really novel, Sun had VIS-powered implementation for a while.
19 # Unlike Sun's implementation this one can process multiple unaligned
20 # input blocks, and as such works as drop-in replacement for OpenSSL
21 # sha1_block_data_order. Performance improvement was measured to be
22 # 40% over pure IALU sha1-sparcv9.pl on UltraSPARC-IIi, but 12% on
23 # UltraSPARC-III. See below for discussion...
25 # (*) "Pipe-lined" means that even if it takes several cycles to
26 # complete, next instruction using same functional unit [but not
27 # depending on the result of the current instruction] can start
28 # execution without having to wait for the unit. "Pairable"
29 # means that two [or more] independent instructions can be
30 # issued at the very same time.
33 for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); }
34 if ($bits==64) { $bias=2047; $frame=192; }
35 else { $bias=0; $frame=112; }
38 open STDOUT,">$output";
72 @VK=($VK_00_19,$VK_20_39,$VK_40_59,$VK_60_79);
73 @X=("%f0", "%f1", "%f2", "%f3", "%f4", "%f5", "%f6", "%f7",
74 "%f8", "%f9","%f10","%f11","%f12","%f13","%f14","%f15","%f16");
76 # This is reference 2x-parallelized VIS-powered Xupdate procedure. It
77 # covers even K_NN_MM addition...
80 my $K=@VK[($i+16)/20];
83 # [ provided that GSR.alignaddr_offset is 5, $mul contains
84 # 0x100ULL<<32|0x100 value and K_NN_MM are pre-loaded to
85 # chosen registers... ]
87 fxors @X[($j+13)%16],@X[$j],@X[$j] !-1/-1/-1:X[0]^=X[13]
88 fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14]
89 fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9]
90 fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9]
91 faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24
92 fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1
93 fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1
95 for %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp
96 ![fxors %f0,%f3,%f3] !10/17/12:X[0] dependency
97 fpadd32 $K,@X[$j],%f20
98 std %f20,[$Xfer+`4*$j`]
100 # The numbers delimited with slash are the earliest possible dispatch
101 # cycles for given instruction assuming 1 cycle latency for simple VIS
102 # instructions, such as on UltraSPARC-I&II, 3 cycles latency, such as
103 # on UltraSPARC-III&IV, and 2 cycles latency, such as on SPARC64-V[?],
104 # respectively. Being 2x-parallelized the procedure is "worth" 5, 8.5
105 # or 6 ticks per SHA1 round. As FPU/VIS instructions are perfectly
106 # pairable with IALU ones, the round timing is defined by the maximum
107 # between VIS and IALU timings. The latter varies from round to round
108 # and averages out at 6.25 ticks. This means that USI&II and SPARC64-V
109 # should operate at IALU rate, while USIII&IV - at VIS rate. This
110 # explains why performance improvement varies among processors. Well,
111 # it should be noted that pure IALU sha1-sparcv9.pl module exhibits
112 # virtually uniform performance of ~9.3 cycles per SHA1 round. Timings
113 # mentioned above are theoretical lower limits. Real-life performance
114 # was measured to be 6.6 cycles per SHA1 round on USIIi and 8.3 on
115 # USIII. The latter is lower than half-round VIS timing, because there
116 # are 16 Xupdate-free rounds, which "push down" average theoretical
117 # timing to 8 cycles...
120 # The reference Xupdate procedure is then "strained" over *pairs* of
121 # BODY_NN_MM and kind of modulo-scheduled in respect to X[n]^=X[n+13]
122 # and K_NN_MM addition. It's "running" 15 rounds ahead, which leaves
123 # plenty of room to amortize for read-after-write hazard, as well as
124 # to fetch and align input for the next spin. The VIS instructions are
125 # scheduled for latency of 2 cycles, because there are not enough IALU
126 # instructions to schedule for latency of 3, while scheduling for 1
127 # would give no gain on USI&II, but loss on SPARC64-V.
130 my ($i,$a,$b,$c,$d,$e)=@_;
132 my $k=($j+16+2)%16; # ahead reference
133 my $l=($j+16-2)%16; # behind reference
134 my $K=@VK[($j+16-2)/20];
138 $code.=<<___ if (!($i&1));
141 ld [$Xfer+`4*($i%16)`],$Xi
142 fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14]
145 fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9]
150 fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9]
155 faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24
157 $code.=<<___ if ($i&1);
160 ld [$Xfer+`4*($i%16)`],$Xi
161 fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1
164 fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1
167 fpadd32 $K,@X[$l],%f20 !
170 fxors @X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X[0]^=X[13]
173 fxor %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp
177 $code.=<<___ if ($i&1 && $i>=2);
178 std %f20,[$Xfer+`4*$l`] !
183 my ($i,$a,$b,$c,$d,$e)=@_;
185 my $k=($j+16+2)%16; # ahead reference
186 my $l=($j+16-2)%16; # behind reference
187 my $K=@VK[($j+16-2)/20];
191 $code.=<<___ if (!($i&1) && $i<64);
193 ld [$Xfer+`4*($i%16)`],$Xi
194 fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14]
197 fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9]
202 fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9]
207 faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24
209 $code.=<<___ if ($i&1 && $i<64);
211 ld [$Xfer+`4*($i%16)`],$Xi
212 fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1
215 fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1
218 fpadd32 $K,@X[$l],%f20 !
221 fxors @X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X[0]^=X[13]
224 fxor %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp
227 std %f20,[$Xfer+`4*$l`] !
229 $code.=<<___ if ($i==64);
231 ld [$Xfer+`4*($i%16)`],$Xi
232 fpadd32 $K,@X[$l],%f20
239 std %f20,[$Xfer+`4*$l`]
245 $code.=<<___ if ($i>64);
247 ld [$Xfer+`4*($i%16)`],$Xi
262 my ($i,$a,$b,$c,$d,$e)=@_;
264 my $k=($j+16+2)%16; # ahead reference
265 my $l=($j+16-2)%16; # behind reference
266 my $K=@VK[($j+16-2)/20];
270 $code.=<<___ if (!($i&1));
272 ld [$Xfer+`4*($i%16)`],$Xi
273 fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14]
276 fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9]
281 fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9]
286 faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24
289 fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1
291 $code.=<<___ if ($i&1);
293 ld [$Xfer+`4*($i%16)`],$Xi
296 fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1
299 fpadd32 $K,@X[$l],%f20 !
302 fxors @X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X[0]^=X[13]
305 fxor %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp
310 std %f20,[$Xfer+`4*$l`] !
314 # If there is more data to process, then we pre-fetch the data for
315 # next iteration in last ten rounds...
317 my ($i,$a,$b,$c,$d,$e)=@_;
323 $code.=<<___ if ($i==70);
325 ld [$Xfer+`4*($i%16)`],$Xi
340 and $nXfer,255,$nXfer
341 alignaddr %g0,$align,%g0
342 add $base,$nXfer,$nXfer
344 $code.=<<___ if ($i==71);
346 ld [$Xfer+`4*($i%16)`],$Xi
358 $code.=<<___ if ($i>=72);
359 faligndata @X[$m],@X[$m+2],@X[$m]
361 ld [$Xfer+`4*($i%16)`],$Xi
366 fpadd32 $VK_00_19,@X[$m],%f20
374 $code.=<<___ if ($i<77);
375 ldd [$inp+`8*($i+1-70)`],@X[2*($i+1-70)]
377 $code.=<<___ if ($i==77); # redundant if $inp was aligned
380 ldd [$inp+$tmp0],@X[16]
382 $code.=<<___ if ($i>=72);
383 std %f20,[$nXfer+`4*$m`]
388 .section ".text",#alloc,#execinstr
392 .long 0x5a827999,0x5a827999 ! K_00_19
393 .long 0x6ed9eba1,0x6ed9eba1 ! K_20_39
394 .long 0x8f1bbcdc,0x8f1bbcdc ! K_40_59
395 .long 0xca62c1d6,0xca62c1d6 ! K_60_79
396 .long 0x00000100,0x00000100
398 .type vis_const,#object
399 .size vis_const,(.-vis_const)
401 ldd [$tmp0+0],$VK_00_19
402 ldd [$tmp0+8],$VK_20_39
403 ldd [$tmp0+16],$VK_40_59
404 ldd [$tmp0+24],$VK_60_79
407 .type load_vis_const,#function
408 .size load_vis_const,(.-load_vis_const)
411 .globl sha1_block_data_order
412 sha1_block_data_order:
414 add %fp,$bias-256,$base
416 1: call load_vis_const
417 sub %o7,1b-vis_const,$tmp0
422 sub $base,$bias+$frame,%sp
429 # X[16] is maintained in FP register bank
430 alignaddr %g0,$align,%g0
438 add $base,$Xfer,$Xfer
442 brz,pt $align,.Laligned
446 faligndata @X[0],@X[2],@X[0]
447 faligndata @X[2],@X[4],@X[2]
448 faligndata @X[4],@X[6],@X[4]
449 faligndata @X[6],@X[8],@X[6]
450 faligndata @X[8],@X[10],@X[8]
451 faligndata @X[10],@X[12],@X[10]
452 faligndata @X[12],@X[14],@X[12]
453 faligndata @X[14],@X[16],@X[14]
458 alignaddr %g0,$tmp0,%g0
459 fpadd32 $VK_00_19,@X[0],%f16
460 fpadd32 $VK_00_19,@X[2],%f18
461 fpadd32 $VK_00_19,@X[4],%f20
462 fpadd32 $VK_00_19,@X[6],%f22
463 fpadd32 $VK_00_19,@X[8],%f24
464 fpadd32 $VK_00_19,@X[10],%f26
465 fpadd32 $VK_00_19,@X[12],%f28
466 fpadd32 $VK_00_19,@X[14],%f30
478 fxors @X[13],@X[0],@X[0]
485 for ($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
486 for (;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
487 for (;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
488 for (;$i<70;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
493 for (;$i<80;$i++) { &BODY_70_79($i,@V); unshift(@V,pop(@V)); }
501 fxors @X[13],@X[0],@X[0]
507 alignaddr %g0,$tmp0,%g0
515 for($i=70;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
531 .type sha1_block_data_order,#function
532 .size sha1_block_data_order,(.-sha1_block_data_order)
533 .asciz "SHA1 block transform for SPARCv9a, CRYPTOGAMS by <appro\@openssl.org>"
536 # Purpose of these subroutines is to explicitly encode VIS instructions,
537 # so that one can compile the module without having to specify VIS
538 # extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a.
539 # Idea is to reserve for option to produce "universal" binary and let
540 # programmer detect if current CPU is VIS capable at run-time.
542 my ($mnemonic,$rs1,$rs2,$rd)=@_;
544 my %visopf = ( "fmul8ulx16" => 0x037,
545 "faligndata" => 0x048,
550 $ref = "$mnemonic\t$rs1,$rs2,$rd";
552 if ($opf=$visopf{$mnemonic}) {
553 foreach ($rs1,$rs2,$rd) {
554 return $ref if (!/%f([0-9]{1,2})/);
557 return $ref if ($1&1);
558 # re-encode for upper double register addressing
563 return sprintf ".word\t0x%08x !%s",
564 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2,
571 my ($mnemonic,$rs1,$rs2,$rd)=@_;
572 my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 );
573 my $ref="$mnemonic\t$rs1,$rs2,$rd";
575 foreach ($rs1,$rs2,$rd) {
576 if (/%([goli])([0-7])/) { $_=$bias{$1}+$2; }
577 else { return $ref; }
579 return sprintf ".word\t0x%08x !%s",
580 0x81b00300|$rd<<25|$rs1<<14|$rs2,
584 $code =~ s/\`([^\`]*)\`/eval $1/gem;
585 $code =~ s/\b(f[^\s]*)\s+(%f[0-9]{1,2}),(%f[0-9]{1,2}),(%f[0-9]{1,2})/
588 $code =~ s/\b(alignaddr)\s+(%[goli][0-7]),(%[goli][0-7]),(%[goli][0-7])/
589 &unalignaddr($1,$2,$3,$4)