2 # Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
17 # SHA256/512 for ARMv8.
19 # Performance in cycles per processed byte and improvement coefficient
20 # over code generated with "default" compiler:
22 # SHA256-hw SHA256(*) SHA512
23 # Apple A7 1.97 10.5 (+33%) 6.73 (-1%(**))
24 # Cortex-A53 2.38 15.5 (+115%) 10.0 (+150%(***))
25 # Cortex-A57 2.31 11.6 (+86%) 7.51 (+260%(***))
26 # Denver 2.01 10.5 (+26%) 6.70 (+8%)
27 # X-Gene 20.0 (+100%) 12.8 (+300%(***))
29 # (*) Software SHA256 results are of lesser relevance, presented
30 # mostly for informational purposes.
31 # (**) The result is a trade-off: it's possible to improve it by
32 # 10% (or by 1 cycle per round), but at the cost of 20% loss
33 # on Cortex-A53 (or by 4 cycles per round).
34 # (***) Super-impressive coefficients over gcc-generated code are
35 # indication of some compiler "pathology", most notably code
36 # generated with -mgeneral-regs-only is significanty faster
37 # and the gap is only 40-90%.
42 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
43 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
44 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
45 die "can't locate arm-xlate.pl";
47 open OUT,"| \"$^X\" $xlate $flavour $output";
50 if ($output =~ /512/) {
70 $func="sha${BITS}_block_data_order";
72 ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
74 @X=map("$reg_t$_",(3..15,0..2));
75 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27));
76 ($t0,$t1,$t2,$t3)=map("$reg_t$_",(16,17,19,28));
79 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
81 my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]);
82 $T0=@X[$i+3] if ($i<11);
84 $code.=<<___ if ($i<16);
86 rev @X[$i],@X[$i] // $i
89 $code.=<<___ if ($i<13 && ($i&1));
90 ldp @X[$i+1],@X[$i+2],[$inp],#2*$SZ
92 $code.=<<___ if ($i==13);
93 ldp @X[14],@X[15],[$inp]
95 $code.=<<___ if ($i>=14);
96 ldr @X[($i-11)&15],[sp,#`$SZ*(($i-11)%4)`]
98 $code.=<<___ if ($i>0 && $i<16);
99 add $a,$a,$t1 // h+=Sigma0(a)
101 $code.=<<___ if ($i>=11);
102 str @X[($i-8)&15],[sp,#`$SZ*(($i-8)%4)`]
104 # While ARMv8 specifies merged rotate-n-logical operation such as
105 # 'eor x,y,z,ror#n', it was found to negatively affect performance
106 # on Apple A7. The reason seems to be that it requires even 'y' to
107 # be available earlier. This means that such merged instruction is
108 # not necessarily best choice on critical path... On the other hand
109 # Cortex-A5x handles merged instructions much better than disjoint
110 # rotate and logical... See (**) footnote above.
111 $code.=<<___ if ($i<15);
112 ror $t0,$e,#$Sigma1[0]
113 add $h,$h,$t2 // h+=K[i]
114 eor $T0,$e,$e,ror#`$Sigma1[2]-$Sigma1[1]`
117 add $h,$h,@X[$i&15] // h+=X[i]
118 orr $t1,$t1,$t2 // Ch(e,f,g)
119 eor $t2,$a,$b // a^b, b^c in next round
120 eor $t0,$t0,$T0,ror#$Sigma1[1] // Sigma1(e)
121 ror $T0,$a,#$Sigma0[0]
122 add $h,$h,$t1 // h+=Ch(e,f,g)
123 eor $t1,$a,$a,ror#`$Sigma0[2]-$Sigma0[1]`
124 add $h,$h,$t0 // h+=Sigma1(e)
125 and $t3,$t3,$t2 // (b^c)&=(a^b)
127 eor $t3,$t3,$b // Maj(a,b,c)
128 eor $t1,$T0,$t1,ror#$Sigma0[1] // Sigma0(a)
129 add $h,$h,$t3 // h+=Maj(a,b,c)
130 ldr $t3,[$Ktbl],#$SZ // *K++, $t2 in next round
131 //add $h,$h,$t1 // h+=Sigma0(a)
133 $code.=<<___ if ($i>=15);
134 ror $t0,$e,#$Sigma1[0]
135 add $h,$h,$t2 // h+=K[i]
136 ror $T1,@X[($j+1)&15],#$sigma0[0]
138 ror $T2,@X[($j+14)&15],#$sigma1[0]
140 ror $T0,$a,#$Sigma0[0]
141 add $h,$h,@X[$i&15] // h+=X[i]
142 eor $t0,$t0,$e,ror#$Sigma1[1]
143 eor $T1,$T1,@X[($j+1)&15],ror#$sigma0[1]
144 orr $t1,$t1,$t2 // Ch(e,f,g)
145 eor $t2,$a,$b // a^b, b^c in next round
146 eor $t0,$t0,$e,ror#$Sigma1[2] // Sigma1(e)
147 eor $T0,$T0,$a,ror#$Sigma0[1]
148 add $h,$h,$t1 // h+=Ch(e,f,g)
149 and $t3,$t3,$t2 // (b^c)&=(a^b)
150 eor $T2,$T2,@X[($j+14)&15],ror#$sigma1[1]
151 eor $T1,$T1,@X[($j+1)&15],lsr#$sigma0[2] // sigma0(X[i+1])
152 add $h,$h,$t0 // h+=Sigma1(e)
153 eor $t3,$t3,$b // Maj(a,b,c)
154 eor $t1,$T0,$a,ror#$Sigma0[2] // Sigma0(a)
155 eor $T2,$T2,@X[($j+14)&15],lsr#$sigma1[2] // sigma1(X[i+14])
156 add @X[$j],@X[$j],@X[($j+9)&15]
158 add $h,$h,$t3 // h+=Maj(a,b,c)
159 ldr $t3,[$Ktbl],#$SZ // *K++, $t2 in next round
160 add @X[$j],@X[$j],$T1
161 add $h,$h,$t1 // h+=Sigma0(a)
162 add @X[$j],@X[$j],$T2
168 #include "arm_arch.h"
172 .extern OPENSSL_armcap_P
174 .type $func,%function
178 $code.=<<___ if ($SZ==4);
180 ldrsw x16,.LOPENSSL_armcap_P
182 ldr x16,.LOPENSSL_armcap_P
184 adr x17,.LOPENSSL_armcap_P
187 tst w16,#ARMV8_SHA256
191 stp x29,x30,[sp,#-128]!
201 ldp $A,$B,[$ctx] // load context
202 ldp $C,$D,[$ctx,#2*$SZ]
203 ldp $E,$F,[$ctx,#4*$SZ]
204 add $num,$inp,$num,lsl#`log(16*$SZ)/log(2)` // end of input
205 ldp $G,$H,[$ctx,#6*$SZ]
207 stp $ctx,$num,[x29,#96]
210 ldp @X[0],@X[1],[$inp],#2*$SZ
211 ldr $t2,[$Ktbl],#$SZ // *K++
212 eor $t3,$B,$C // magic seed
215 for ($i=0;$i<16;$i++) { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
216 $code.=".Loop_16_xx:\n";
217 for (;$i<32;$i++) { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
221 ldp $ctx,$num,[x29,#96]
223 sub $Ktbl,$Ktbl,#`$SZ*($rounds+1)` // rewind
225 ldp @X[0],@X[1],[$ctx]
226 ldp @X[2],@X[3],[$ctx,#2*$SZ]
227 add $inp,$inp,#14*$SZ // advance input pointer
228 ldp @X[4],@X[5],[$ctx,#4*$SZ]
230 ldp @X[6],@X[7],[$ctx,#6*$SZ]
237 stp $C,$D,[$ctx,#2*$SZ]
241 stp $E,$F,[$ctx,#4*$SZ]
242 stp $G,$H,[$ctx,#6*$SZ]
245 ldp x19,x20,[x29,#16]
247 ldp x21,x22,[x29,#32]
248 ldp x23,x24,[x29,#48]
249 ldp x25,x26,[x29,#64]
250 ldp x27,x28,[x29,#80]
251 ldp x29,x30,[sp],#128
256 .type .LK$BITS,%object
259 $code.=<<___ if ($SZ==8);
260 .quad 0x428a2f98d728ae22,0x7137449123ef65cd
261 .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
262 .quad 0x3956c25bf348b538,0x59f111f1b605d019
263 .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
264 .quad 0xd807aa98a3030242,0x12835b0145706fbe
265 .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
266 .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
267 .quad 0x9bdc06a725c71235,0xc19bf174cf692694
268 .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
269 .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
270 .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
271 .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
272 .quad 0x983e5152ee66dfab,0xa831c66d2db43210
273 .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
274 .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
275 .quad 0x06ca6351e003826f,0x142929670a0e6e70
276 .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
277 .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
278 .quad 0x650a73548baf63de,0x766a0abb3c77b2a8
279 .quad 0x81c2c92e47edaee6,0x92722c851482353b
280 .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
281 .quad 0xc24b8b70d0f89791,0xc76c51a30654be30
282 .quad 0xd192e819d6ef5218,0xd69906245565a910
283 .quad 0xf40e35855771202a,0x106aa07032bbd1b8
284 .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
285 .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
286 .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
287 .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
288 .quad 0x748f82ee5defb2fc,0x78a5636f43172f60
289 .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
290 .quad 0x90befffa23631e28,0xa4506cebde82bde9
291 .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
292 .quad 0xca273eceea26619c,0xd186b8c721c0c207
293 .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
294 .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
295 .quad 0x113f9804bef90dae,0x1b710b35131c471b
296 .quad 0x28db77f523047d84,0x32caab7b40c72493
297 .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
298 .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
299 .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
300 .quad 0 // terminator
302 $code.=<<___ if ($SZ==4);
303 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
304 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
305 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
306 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
307 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
308 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
309 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
310 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
311 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
312 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
313 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
314 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
315 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
316 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
317 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
318 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
322 .size .LK$BITS,.-.LK$BITS
326 .long OPENSSL_armcap_P-.
328 .quad OPENSSL_armcap_P-.
330 .asciz "SHA$BITS block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
337 my ($ABCD,$EFGH,$abcd)=map("v$_.16b",(0..2));
338 my @MSG=map("v$_.16b",(4..7));
339 my ($W0,$W1)=("v16.4s","v17.4s");
340 my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
343 .type sha256_block_armv8,%function
347 stp x29,x30,[sp,#-16]!
350 ld1.32 {$ABCD,$EFGH},[$ctx]
354 ld1 {@MSG[0]-@MSG[3]},[$inp],#64
356 ld1.32 {$W0},[$Ktbl],#16
357 rev32 @MSG[0],@MSG[0]
358 rev32 @MSG[1],@MSG[1]
359 rev32 @MSG[2],@MSG[2]
360 rev32 @MSG[3],@MSG[3]
361 orr $ABCD_SAVE,$ABCD,$ABCD // offload
362 orr $EFGH_SAVE,$EFGH,$EFGH
364 for($i=0;$i<12;$i++) {
366 ld1.32 {$W1},[$Ktbl],#16
367 add.i32 $W0,$W0,@MSG[0]
368 sha256su0 @MSG[0],@MSG[1]
369 orr $abcd,$ABCD,$ABCD
370 sha256h $ABCD,$EFGH,$W0
371 sha256h2 $EFGH,$abcd,$W0
372 sha256su1 @MSG[0],@MSG[2],@MSG[3]
374 ($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG));
377 ld1.32 {$W1},[$Ktbl],#16
378 add.i32 $W0,$W0,@MSG[0]
379 orr $abcd,$ABCD,$ABCD
380 sha256h $ABCD,$EFGH,$W0
381 sha256h2 $EFGH,$abcd,$W0
383 ld1.32 {$W0},[$Ktbl],#16
384 add.i32 $W1,$W1,@MSG[1]
385 orr $abcd,$ABCD,$ABCD
386 sha256h $ABCD,$EFGH,$W1
387 sha256h2 $EFGH,$abcd,$W1
390 add.i32 $W0,$W0,@MSG[2]
391 sub $Ktbl,$Ktbl,#$rounds*$SZ-16 // rewind
392 orr $abcd,$ABCD,$ABCD
393 sha256h $ABCD,$EFGH,$W0
394 sha256h2 $EFGH,$abcd,$W0
396 add.i32 $W1,$W1,@MSG[3]
397 orr $abcd,$ABCD,$ABCD
398 sha256h $ABCD,$EFGH,$W1
399 sha256h2 $EFGH,$abcd,$W1
401 add.i32 $ABCD,$ABCD,$ABCD_SAVE
402 add.i32 $EFGH,$EFGH,$EFGH_SAVE
406 st1.32 {$ABCD,$EFGH},[$ctx]
410 .size sha256_block_armv8,.-sha256_block_armv8
415 .comm OPENSSL_armcap_P,4,4
419 "sha256h" => 0x5e004000, "sha256h2" => 0x5e005000,
420 "sha256su0" => 0x5e282800, "sha256su1" => 0x5e006000 );
423 my ($mnemonic,$arg)=@_;
425 $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
427 sprintf ".inst\t0x%08x\t//%s %s",
428 $opcode{$mnemonic}|$1|($2<<5)|($3<<16),
433 foreach(split("\n",$code)) {
435 s/\`([^\`]*)\`/eval($1)/geo;
437 s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/geo;
439 s/\.\w?32\b//o and s/\.16b/\.4s/go;
440 m/(ld|st)1[^\[]+\[0\]/o and s/\.4s/\.s/go;