crypto/sha/asm/sha512-armv8.pl

   1 #!/usr/bin/env perl
   2 #
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9 #
  10 # SHA256/512 for ARMv8.
  11 #
  12 # Performance in cycles per processed byte and improvement coefficient
  13 # over code generated with "default" compiler:
  14 #
  15 #               SHA256-hw       SHA256(*)       SHA512
  16 # Apple A7      1.97            10.5 (+33%)     6.73 (-1%(**))
  17 # Cortex-A53    2.38            15.5 (+115%)    10.0 (+150%(***))
  18 # Cortex-A57    2.31            11.6 (+86%)     7.51 (+260%(***))
  19 # Denver        2.01            10.5 (+26%)     6.70 (+8%)
  20 # X-Gene                        20.0 (+100%)    12.8 (+300%(***))
  21 #
  22 # (*)   Software SHA256 results are of lesser relevance, presented
  23 #       mostly for informational purposes.
  24 # (**)  The result is a trade-off: it's possible to improve it by
  25 #       10% (or by 1 cycle per round), but at the cost of 20% loss
  26 #       on Cortex-A53 (or by 4 cycles per round).
  27 # (***) Super-impressive coefficients over gcc-generated code are
  28 #       indication of some compiler "pathology", most notably code
  29 #       generated with -mgeneral-regs-only is significanty faster
  30 #       and the gap is only 40-90%.
  31
  32 $flavour=shift;
  33 $output=shift;
  34
  35 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  36 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  37 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  38 die "can't locate arm-xlate.pl";
  39
  40 open OUT,"| \"$^X\" $xlate $flavour $output";
  41 *STDOUT=*OUT;
  42
  43 if ($output =~ /512/) {
  44         $BITS=512;
  45         $SZ=8;
  46         @Sigma0=(28,34,39);
  47         @Sigma1=(14,18,41);
  48         @sigma0=(1,  8, 7);
  49         @sigma1=(19,61, 6);
  50         $rounds=80;
  51         $reg_t="x";
  52 } else {
  53         $BITS=256;
  54         $SZ=4;
  55         @Sigma0=( 2,13,22);
  56         @Sigma1=( 6,11,25);
  57         @sigma0=( 7,18, 3);
  58         @sigma1=(17,19,10);
  59         $rounds=64;
  60         $reg_t="w";
  61 }
  62
  63 $func="sha${BITS}_block_data_order";
  64
  65 ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
  66
  67 @X=map("$reg_t$_",(3..15,0..2));
  68 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27));
  69 ($t0,$t1,$t2,$t3)=map("$reg_t$_",(16,17,19,28));
  70
  71 sub BODY_00_xx {
  72 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
  73 my $j=($i+1)&15;
  74 my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]);
  75    $T0=@X[$i+3] if ($i<11);
  76
  77 $code.=<<___    if ($i<16);
  78 #ifndef __ARMEB__
  79         rev     @X[$i],@X[$i]                   // $i
  80 #endif
  81 ___
  82 $code.=<<___    if ($i<13 && ($i&1));
  83         ldp     @X[$i+1],@X[$i+2],[$inp],#2*$SZ
  84 ___
  85 $code.=<<___    if ($i==13);
  86         ldp     @X[14],@X[15],[$inp]
  87 ___
  88 $code.=<<___    if ($i>=14);
  89         ldr     @X[($i-11)&15],[sp,#`$SZ*(($i-11)%4)`]
  90 ___
  91 $code.=<<___    if ($i>0 && $i<16);
  92         add     $a,$a,$t1                       // h+=Sigma0(a)
  93 ___
  94 $code.=<<___    if ($i>=11);
  95         str     @X[($i-8)&15],[sp,#`$SZ*(($i-8)%4)`]
  96 ___
  97 # While ARMv8 specifies merged rotate-n-logical operation such as
  98 # 'eor x,y,z,ror#n', it was found to negatively affect performance
  99 # on Apple A7. The reason seems to be that it requires even 'y' to
 100 # be available earlier. This means that such merged instruction is
 101 # not necessarily best choice on critical path... On the other hand
 102 # Cortex-A5x handles merged instructions much better than disjoint
 103 # rotate and logical... See (**) footnote above.
 104 $code.=<<___    if ($i<15);
 105         ror     $t0,$e,#$Sigma1[0]
 106         add     $h,$h,$t2                       // h+=K[i]
 107         eor     $T0,$e,$e,ror#`$Sigma1[2]-$Sigma1[1]`
 108         and     $t1,$f,$e
 109         bic     $t2,$g,$e
 110         add     $h,$h,@X[$i&15]                 // h+=X[i]
 111         orr     $t1,$t1,$t2                     // Ch(e,f,g)
 112         eor     $t2,$a,$b                       // a^b, b^c in next round
 113         eor     $t0,$t0,$T0,ror#$Sigma1[1]      // Sigma1(e)
 114         ror     $T0,$a,#$Sigma0[0]
 115         add     $h,$h,$t1                       // h+=Ch(e,f,g)
 116         eor     $t1,$a,$a,ror#`$Sigma0[2]-$Sigma0[1]`
 117         add     $h,$h,$t0                       // h+=Sigma1(e)
 118         and     $t3,$t3,$t2                     // (b^c)&=(a^b)
 119         add     $d,$d,$h                        // d+=h
 120         eor     $t3,$t3,$b                      // Maj(a,b,c)
 121         eor     $t1,$T0,$t1,ror#$Sigma0[1]      // Sigma0(a)
 122         add     $h,$h,$t3                       // h+=Maj(a,b,c)
 123         ldr     $t3,[$Ktbl],#$SZ                // *K++, $t2 in next round
 124         //add   $h,$h,$t1                       // h+=Sigma0(a)
 125 ___
 126 $code.=<<___    if ($i>=15);
 127         ror     $t0,$e,#$Sigma1[0]
 128         add     $h,$h,$t2                       // h+=K[i]
 129         ror     $T1,@X[($j+1)&15],#$sigma0[0]
 130         and     $t1,$f,$e
 131         ror     $T2,@X[($j+14)&15],#$sigma1[0]
 132         bic     $t2,$g,$e
 133         ror     $T0,$a,#$Sigma0[0]
 134         add     $h,$h,@X[$i&15]                 // h+=X[i]
 135         eor     $t0,$t0,$e,ror#$Sigma1[1]
 136         eor     $T1,$T1,@X[($j+1)&15],ror#$sigma0[1]
 137         orr     $t1,$t1,$t2                     // Ch(e,f,g)
 138         eor     $t2,$a,$b                       // a^b, b^c in next round
 139         eor     $t0,$t0,$e,ror#$Sigma1[2]       // Sigma1(e)
 140         eor     $T0,$T0,$a,ror#$Sigma0[1]
 141         add     $h,$h,$t1                       // h+=Ch(e,f,g)
 142         and     $t3,$t3,$t2                     // (b^c)&=(a^b)
 143         eor     $T2,$T2,@X[($j+14)&15],ror#$sigma1[1]
 144         eor     $T1,$T1,@X[($j+1)&15],lsr#$sigma0[2]    // sigma0(X[i+1])
 145         add     $h,$h,$t0                       // h+=Sigma1(e)
 146         eor     $t3,$t3,$b                      // Maj(a,b,c)
 147         eor     $t1,$T0,$a,ror#$Sigma0[2]       // Sigma0(a)
 148         eor     $T2,$T2,@X[($j+14)&15],lsr#$sigma1[2]   // sigma1(X[i+14])
 149         add     @X[$j],@X[$j],@X[($j+9)&15]
 150         add     $d,$d,$h                        // d+=h
 151         add     $h,$h,$t3                       // h+=Maj(a,b,c)
 152         ldr     $t3,[$Ktbl],#$SZ                // *K++, $t2 in next round
 153         add     @X[$j],@X[$j],$T1
 154         add     $h,$h,$t1                       // h+=Sigma0(a)
 155         add     @X[$j],@X[$j],$T2
 156 ___
 157         ($t2,$t3)=($t3,$t2);
 158 }
 159
 160 $code.=<<___;
 161 #include "arm_arch.h"
 162
 163 .text
 164
 165 .extern OPENSSL_armcap_P
 166 .globl  $func
 167 .type   $func,%function
 168 .align  6
 169 $func:
 170 ___
 171 $code.=<<___    if ($SZ==4);
 172         ldr     x16,.LOPENSSL_armcap_P
 173         adr     x17,.LOPENSSL_armcap_P
 174         add     x16,x16,x17
 175         ldr     w16,[x16]
 176         tst     w16,#ARMV8_SHA256
 177         b.ne    .Lv8_entry
 178 ___
 179 $code.=<<___;
 180         stp     x29,x30,[sp,#-128]!
 181         add     x29,sp,#0
 182
 183         stp     x19,x20,[sp,#16]
 184         stp     x21,x22,[sp,#32]
 185         stp     x23,x24,[sp,#48]
 186         stp     x25,x26,[sp,#64]
 187         stp     x27,x28,[sp,#80]
 188         sub     sp,sp,#4*$SZ
 189
 190         ldp     $A,$B,[$ctx]                            // load context
 191         ldp     $C,$D,[$ctx,#2*$SZ]
 192         ldp     $E,$F,[$ctx,#4*$SZ]
 193         add     $num,$inp,$num,lsl#`log(16*$SZ)/log(2)` // end of input
 194         ldp     $G,$H,[$ctx,#6*$SZ]
 195         adr     $Ktbl,.LK$BITS
 196         stp     $ctx,$num,[x29,#96]
 197
 198 .Loop:
 199         ldp     @X[0],@X[1],[$inp],#2*$SZ
 200         ldr     $t2,[$Ktbl],#$SZ                        // *K++
 201         eor     $t3,$B,$C                               // magic seed
 202         str     $inp,[x29,#112]
 203 ___
 204 for ($i=0;$i<16;$i++)   { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
 205 $code.=".Loop_16_xx:\n";
 206 for (;$i<32;$i++)       { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
 207 $code.=<<___;
 208         cbnz    $t2,.Loop_16_xx
 209
 210         ldp     $ctx,$num,[x29,#96]
 211         ldr     $inp,[x29,#112]
 212         sub     $Ktbl,$Ktbl,#`$SZ*($rounds+1)`          // rewind
 213
 214         ldp     @X[0],@X[1],[$ctx]
 215         ldp     @X[2],@X[3],[$ctx,#2*$SZ]
 216         add     $inp,$inp,#14*$SZ                       // advance input pointer
 217         ldp     @X[4],@X[5],[$ctx,#4*$SZ]
 218         add     $A,$A,@X[0]
 219         ldp     @X[6],@X[7],[$ctx,#6*$SZ]
 220         add     $B,$B,@X[1]
 221         add     $C,$C,@X[2]
 222         add     $D,$D,@X[3]
 223         stp     $A,$B,[$ctx]
 224         add     $E,$E,@X[4]
 225         add     $F,$F,@X[5]
 226         stp     $C,$D,[$ctx,#2*$SZ]
 227         add     $G,$G,@X[6]
 228         add     $H,$H,@X[7]
 229         cmp     $inp,$num
 230         stp     $E,$F,[$ctx,#4*$SZ]
 231         stp     $G,$H,[$ctx,#6*$SZ]
 232         b.ne    .Loop
 233
 234         ldp     x19,x20,[x29,#16]
 235         add     sp,sp,#4*$SZ
 236         ldp     x21,x22,[x29,#32]
 237         ldp     x23,x24,[x29,#48]
 238         ldp     x25,x26,[x29,#64]
 239         ldp     x27,x28,[x29,#80]
 240         ldp     x29,x30,[sp],#128
 241         ret
 242 .size   $func,.-$func
 243
 244 .align  6
 245 .type   .LK$BITS,%object
 246 .LK$BITS:
 247 ___
 248 $code.=<<___ if ($SZ==8);
 249         .quad   0x428a2f98d728ae22,0x7137449123ef65cd
 250         .quad   0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
 251         .quad   0x3956c25bf348b538,0x59f111f1b605d019
 252         .quad   0x923f82a4af194f9b,0xab1c5ed5da6d8118
 253         .quad   0xd807aa98a3030242,0x12835b0145706fbe
 254         .quad   0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
 255         .quad   0x72be5d74f27b896f,0x80deb1fe3b1696b1
 256         .quad   0x9bdc06a725c71235,0xc19bf174cf692694
 257         .quad   0xe49b69c19ef14ad2,0xefbe4786384f25e3
 258         .quad   0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
 259         .quad   0x2de92c6f592b0275,0x4a7484aa6ea6e483
 260         .quad   0x5cb0a9dcbd41fbd4,0x76f988da831153b5
 261         .quad   0x983e5152ee66dfab,0xa831c66d2db43210
 262         .quad   0xb00327c898fb213f,0xbf597fc7beef0ee4
 263         .quad   0xc6e00bf33da88fc2,0xd5a79147930aa725
 264         .quad   0x06ca6351e003826f,0x142929670a0e6e70
 265         .quad   0x27b70a8546d22ffc,0x2e1b21385c26c926
 266         .quad   0x4d2c6dfc5ac42aed,0x53380d139d95b3df
 267         .quad   0x650a73548baf63de,0x766a0abb3c77b2a8
 268         .quad   0x81c2c92e47edaee6,0x92722c851482353b
 269         .quad   0xa2bfe8a14cf10364,0xa81a664bbc423001
 270         .quad   0xc24b8b70d0f89791,0xc76c51a30654be30
 271         .quad   0xd192e819d6ef5218,0xd69906245565a910
 272         .quad   0xf40e35855771202a,0x106aa07032bbd1b8
 273         .quad   0x19a4c116b8d2d0c8,0x1e376c085141ab53
 274         .quad   0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
 275         .quad   0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
 276         .quad   0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
 277         .quad   0x748f82ee5defb2fc,0x78a5636f43172f60
 278         .quad   0x84c87814a1f0ab72,0x8cc702081a6439ec
 279         .quad   0x90befffa23631e28,0xa4506cebde82bde9
 280         .quad   0xbef9a3f7b2c67915,0xc67178f2e372532b
 281         .quad   0xca273eceea26619c,0xd186b8c721c0c207
 282         .quad   0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
 283         .quad   0x06f067aa72176fba,0x0a637dc5a2c898a6
 284         .quad   0x113f9804bef90dae,0x1b710b35131c471b
 285         .quad   0x28db77f523047d84,0x32caab7b40c72493
 286         .quad   0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
 287         .quad   0x4cc5d4becb3e42b6,0x597f299cfc657e2a
 288         .quad   0x5fcb6fab3ad6faec,0x6c44198c4a475817
 289         .quad   0       // terminator
 290 ___
 291 $code.=<<___ if ($SZ==4);
 292         .long   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
 293         .long   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
 294         .long   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
 295         .long   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
 296         .long   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
 297         .long   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
 298         .long   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
 299         .long   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
 300         .long   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
 301         .long   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
 302         .long   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
 303         .long   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
 304         .long   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
 305         .long   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
 306         .long   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
 307         .long   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
 308         .long   0       //terminator
 309 ___
 310 $code.=<<___;
 311 .size   .LK$BITS,.-.LK$BITS
 312 .align  3
 313 .LOPENSSL_armcap_P:
 314         .quad   OPENSSL_armcap_P-.
 315 .asciz  "SHA$BITS block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
 316 .align  2
 317 ___
 318
 319 if ($SZ==4) {
 320 my $Ktbl="x3";
 321
 322 my ($ABCD,$EFGH,$abcd)=map("v$_.16b",(0..2));
 323 my @MSG=map("v$_.16b",(4..7));
 324 my ($W0,$W1)=("v16.4s","v17.4s");
 325 my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
 326
 327 $code.=<<___;
 328 .type   sha256_block_armv8,%function
 329 .align  6
 330 sha256_block_armv8:
 331 .Lv8_entry:
 332         stp             x29,x30,[sp,#-16]!
 333         add             x29,sp,#0
 334
 335         ld1.32          {$ABCD,$EFGH},[$ctx]
 336         adr             $Ktbl,.LK256
 337
 338 .Loop_hw:
 339         ld1             {@MSG[0]-@MSG[3]},[$inp],#64
 340         sub             $num,$num,#1
 341         ld1.32          {$W0},[$Ktbl],#16
 342         rev32           @MSG[0],@MSG[0]
 343         rev32           @MSG[1],@MSG[1]
 344         rev32           @MSG[2],@MSG[2]
 345         rev32           @MSG[3],@MSG[3]
 346         orr             $ABCD_SAVE,$ABCD,$ABCD          // offload
 347         orr             $EFGH_SAVE,$EFGH,$EFGH
 348 ___
 349 for($i=0;$i<12;$i++) {
 350 $code.=<<___;
 351         ld1.32          {$W1},[$Ktbl],#16
 352         add.i32         $W0,$W0,@MSG[0]
 353         sha256su0       @MSG[0],@MSG[1]
 354         orr             $abcd,$ABCD,$ABCD
 355         sha256h         $ABCD,$EFGH,$W0
 356         sha256h2        $EFGH,$abcd,$W0
 357         sha256su1       @MSG[0],@MSG[2],@MSG[3]
 358 ___
 359         ($W0,$W1)=($W1,$W0);    push(@MSG,shift(@MSG));
 360 }
 361 $code.=<<___;
 362         ld1.32          {$W1},[$Ktbl],#16
 363         add.i32         $W0,$W0,@MSG[0]
 364         orr             $abcd,$ABCD,$ABCD
 365         sha256h         $ABCD,$EFGH,$W0
 366         sha256h2        $EFGH,$abcd,$W0
 367
 368         ld1.32          {$W0},[$Ktbl],#16
 369         add.i32         $W1,$W1,@MSG[1]
 370         orr             $abcd,$ABCD,$ABCD
 371         sha256h         $ABCD,$EFGH,$W1
 372         sha256h2        $EFGH,$abcd,$W1
 373
 374         ld1.32          {$W1},[$Ktbl]
 375         add.i32         $W0,$W0,@MSG[2]
 376         sub             $Ktbl,$Ktbl,#$rounds*$SZ-16     // rewind
 377         orr             $abcd,$ABCD,$ABCD
 378         sha256h         $ABCD,$EFGH,$W0
 379         sha256h2        $EFGH,$abcd,$W0
 380
 381         add.i32         $W1,$W1,@MSG[3]
 382         orr             $abcd,$ABCD,$ABCD
 383         sha256h         $ABCD,$EFGH,$W1
 384         sha256h2        $EFGH,$abcd,$W1
 385
 386         add.i32         $ABCD,$ABCD,$ABCD_SAVE
 387         add.i32         $EFGH,$EFGH,$EFGH_SAVE
 388
 389         cbnz            $num,.Loop_hw
 390
 391         st1.32          {$ABCD,$EFGH},[$ctx]
 392
 393         ldr             x29,[sp],#16
 394         ret
 395 .size   sha256_block_armv8,.-sha256_block_armv8
 396 ___
 397 }
 398
 399 $code.=<<___;
 400 .comm   OPENSSL_armcap_P,4,4
 401 ___
 402
 403 {   my  %opcode = (
 404         "sha256h"       => 0x5e004000,  "sha256h2"      => 0x5e005000,
 405         "sha256su0"     => 0x5e282800,  "sha256su1"     => 0x5e006000   );
 406
 407     sub unsha256 {
 408         my ($mnemonic,$arg)=@_;
 409
 410         $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
 411         &&
 412         sprintf ".inst\t0x%08x\t//%s %s",
 413                         $opcode{$mnemonic}|$1|($2<<5)|($3<<16),
 414                         $mnemonic,$arg;
 415     }
 416 }
 417
 418 foreach(split("\n",$code)) {
 419
 420         s/\`([^\`]*)\`/eval($1)/geo;
 421
 422         s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/geo;
 423
 424         s/\.\w?32\b//o          and s/\.16b/\.4s/go;
 425         m/(ld|st)1[^\[]+\[0\]/o and s/\.4s/\.s/go;
 426
 427         print $_,"\n";
 428 }
 429
 430 close STDOUT;