crypto/sha/asm/sha512-armv8.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License 2.0 (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9 # ====================================================================
  10 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  11 # project. The module is, however, dual licensed under OpenSSL and
  12 # CRYPTOGAMS licenses depending on where you obtain it. For further
  13 # details see http://www.openssl.org/~appro/cryptogams/.
  14 #
  15 # Permission to use under GPLv2 terms is granted.
  16 # ====================================================================
  17 #
  18 # SHA256/512 for ARMv8.
  19 #
  20 # Performance in cycles per processed byte and improvement coefficient
  21 # over code generated with "default" compiler:
  22 #
  23 #               SHA256-hw       SHA256(*)       SHA512
  24 # Apple A7      1.97            10.5 (+33%)     6.73 (-1%(**))
  25 # Cortex-A53    2.38            15.5 (+115%)    10.0 (+150%(***))
  26 # Cortex-A57    2.31            11.6 (+86%)     7.51 (+260%(***))
  27 # Denver        2.01            10.5 (+26%)     6.70 (+8%)
  28 # X-Gene                        20.0 (+100%)    12.8 (+300%(***))
  29 # Mongoose      2.36            13.0 (+50%)     8.36 (+33%)
  30 # Kryo          1.92            17.4 (+30%)     11.2 (+8%)
  31 # ThunderX2     2.54            13.2 (+40%)     8.40 (+18%)
  32 #
  33 # (*)   Software SHA256 results are of lesser relevance, presented
  34 #       mostly for informational purposes.
  35 # (**)  The result is a trade-off: it's possible to improve it by
  36 #       10% (or by 1 cycle per round), but at the cost of 20% loss
  37 #       on Cortex-A53 (or by 4 cycles per round).
  38 # (***) Super-impressive coefficients over gcc-generated code are
  39 #       indication of some compiler "pathology", most notably code
  40 #       generated with -mgeneral-regs-only is significantly faster
  41 #       and the gap is only 40-90%.
  42 #
  43 # October 2016.
  44 #
  45 # Originally it was reckoned that it makes no sense to implement NEON
  46 # version of SHA256 for 64-bit processors. This is because performance
  47 # improvement on most wide-spread Cortex-A5x processors was observed
  48 # to be marginal, same on Cortex-A53 and ~10% on A57. But then it was
  49 # observed that 32-bit NEON SHA256 performs significantly better than
  50 # 64-bit scalar version on *some* of the more recent processors. As
  51 # result 64-bit NEON version of SHA256 was added to provide best
  52 # all-round performance. For example it executes ~30% faster on X-Gene
  53 # and Mongoose. [For reference, NEON version of SHA512 is bound to
  54 # deliver much less improvement, likely *negative* on Cortex-A5x.
  55 # Which is why NEON support is limited to SHA256.]
  56
  57 # $output is the last argument if it looks like a file (it has an extension)
  58 # $flavour is the first argument if it doesn't look like a file
  59 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  60 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  61
  62 if ($flavour && $flavour ne "void") {
  63     $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  64     ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  65     ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  66     die "can't locate arm-xlate.pl";
  67
  68     open OUT,"| \"$^X\" $xlate $flavour \"$output\""
  69         or die "can't call $xlate: $!";
  70     *STDOUT=*OUT;
  71 } else {
  72     $output and open STDOUT,">$output";
  73 }
  74
  75 if ($output =~ /512/) {
  76         $BITS=512;
  77         $SZ=8;
  78         @Sigma0=(28,34,39);
  79         @Sigma1=(14,18,41);
  80         @sigma0=(1,  8, 7);
  81         @sigma1=(19,61, 6);
  82         $rounds=80;
  83         $reg_t="x";
  84 } else {
  85         $BITS=256;
  86         $SZ=4;
  87         @Sigma0=( 2,13,22);
  88         @Sigma1=( 6,11,25);
  89         @sigma0=( 7,18, 3);
  90         @sigma1=(17,19,10);
  91         $rounds=64;
  92         $reg_t="w";
  93 }
  94
  95 $func="sha${BITS}_block_data_order";
  96
  97 ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
  98
  99 @X=map("$reg_t$_",(3..15,0..2));
 100 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27));
 101 ($t0,$t1,$t2,$t3)=map("$reg_t$_",(16,17,19,28));
 102
 103 sub BODY_00_xx {
 104 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
 105 my $j=($i+1)&15;
 106 my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]);
 107    $T0=@X[$i+3] if ($i<11);
 108
 109 $code.=<<___    if ($i<16);
 110 #ifndef __AARCH64EB__
 111         rev     @X[$i],@X[$i]                   // $i
 112 #endif
 113 ___
 114 $code.=<<___    if ($i<13 && ($i&1));
 115         ldp     @X[$i+1],@X[$i+2],[$inp],#2*$SZ
 116 ___
 117 $code.=<<___    if ($i==13);
 118         ldp     @X[14],@X[15],[$inp]
 119 ___
 120 $code.=<<___    if ($i>=14);
 121         ldr     @X[($i-11)&15],[sp,#`$SZ*(($i-11)%4)`]
 122 ___
 123 $code.=<<___    if ($i>0 && $i<16);
 124         add     $a,$a,$t1                       // h+=Sigma0(a)
 125 ___
 126 $code.=<<___    if ($i>=11);
 127         str     @X[($i-8)&15],[sp,#`$SZ*(($i-8)%4)`]
 128 ___
 129 # While ARMv8 specifies merged rotate-n-logical operation such as
 130 # 'eor x,y,z,ror#n', it was found to negatively affect performance
 131 # on Apple A7. The reason seems to be that it requires even 'y' to
 132 # be available earlier. This means that such merged instruction is
 133 # not necessarily best choice on critical path... On the other hand
 134 # Cortex-A5x handles merged instructions much better than disjoint
 135 # rotate and logical... See (**) footnote above.
 136 $code.=<<___    if ($i<15);
 137         ror     $t0,$e,#$Sigma1[0]
 138         add     $h,$h,$t2                       // h+=K[i]
 139         eor     $T0,$e,$e,ror#`$Sigma1[2]-$Sigma1[1]`
 140         and     $t1,$f,$e
 141         bic     $t2,$g,$e
 142         add     $h,$h,@X[$i&15]                 // h+=X[i]
 143         orr     $t1,$t1,$t2                     // Ch(e,f,g)
 144         eor     $t2,$a,$b                       // a^b, b^c in next round
 145         eor     $t0,$t0,$T0,ror#$Sigma1[1]      // Sigma1(e)
 146         ror     $T0,$a,#$Sigma0[0]
 147         add     $h,$h,$t1                       // h+=Ch(e,f,g)
 148         eor     $t1,$a,$a,ror#`$Sigma0[2]-$Sigma0[1]`
 149         add     $h,$h,$t0                       // h+=Sigma1(e)
 150         and     $t3,$t3,$t2                     // (b^c)&=(a^b)
 151         add     $d,$d,$h                        // d+=h
 152         eor     $t3,$t3,$b                      // Maj(a,b,c)
 153         eor     $t1,$T0,$t1,ror#$Sigma0[1]      // Sigma0(a)
 154         add     $h,$h,$t3                       // h+=Maj(a,b,c)
 155         ldr     $t3,[$Ktbl],#$SZ                // *K++, $t2 in next round
 156         //add   $h,$h,$t1                       // h+=Sigma0(a)
 157 ___
 158 $code.=<<___    if ($i>=15);
 159         ror     $t0,$e,#$Sigma1[0]
 160         add     $h,$h,$t2                       // h+=K[i]
 161         ror     $T1,@X[($j+1)&15],#$sigma0[0]
 162         and     $t1,$f,$e
 163         ror     $T2,@X[($j+14)&15],#$sigma1[0]
 164         bic     $t2,$g,$e
 165         ror     $T0,$a,#$Sigma0[0]
 166         add     $h,$h,@X[$i&15]                 // h+=X[i]
 167         eor     $t0,$t0,$e,ror#$Sigma1[1]
 168         eor     $T1,$T1,@X[($j+1)&15],ror#$sigma0[1]
 169         orr     $t1,$t1,$t2                     // Ch(e,f,g)
 170         eor     $t2,$a,$b                       // a^b, b^c in next round
 171         eor     $t0,$t0,$e,ror#$Sigma1[2]       // Sigma1(e)
 172         eor     $T0,$T0,$a,ror#$Sigma0[1]
 173         add     $h,$h,$t1                       // h+=Ch(e,f,g)
 174         and     $t3,$t3,$t2                     // (b^c)&=(a^b)
 175         eor     $T2,$T2,@X[($j+14)&15],ror#$sigma1[1]
 176         eor     $T1,$T1,@X[($j+1)&15],lsr#$sigma0[2]    // sigma0(X[i+1])
 177         add     $h,$h,$t0                       // h+=Sigma1(e)
 178         eor     $t3,$t3,$b                      // Maj(a,b,c)
 179         eor     $t1,$T0,$a,ror#$Sigma0[2]       // Sigma0(a)
 180         eor     $T2,$T2,@X[($j+14)&15],lsr#$sigma1[2]   // sigma1(X[i+14])
 181         add     @X[$j],@X[$j],@X[($j+9)&15]
 182         add     $d,$d,$h                        // d+=h
 183         add     $h,$h,$t3                       // h+=Maj(a,b,c)
 184         ldr     $t3,[$Ktbl],#$SZ                // *K++, $t2 in next round
 185         add     @X[$j],@X[$j],$T1
 186         add     $h,$h,$t1                       // h+=Sigma0(a)
 187         add     @X[$j],@X[$j],$T2
 188 ___
 189         ($t2,$t3)=($t3,$t2);
 190 }
 191
 192 $code.=<<___;
 193 #include "arm_arch.h"
 194 #ifndef __KERNEL__
 195 .extern OPENSSL_armcap_P
 196 .hidden OPENSSL_armcap_P
 197 #endif
 198
 199 .text
 200
 201 .globl  $func
 202 .type   $func,%function
 203 .align  6
 204 $func:
 205         AARCH64_VALID_CALL_TARGET
 206 #ifndef __KERNEL__
 207         adrp    x16,OPENSSL_armcap_P
 208         ldr     w16,[x16,#:lo12:OPENSSL_armcap_P]
 209 ___
 210 $code.=<<___    if ($SZ==4);
 211         tst     w16,#ARMV8_SHA256
 212         b.ne    .Lv8_entry
 213         tst     w16,#ARMV7_NEON
 214         b.ne    .Lneon_entry
 215 ___
 216 $code.=<<___    if ($SZ==8);
 217         tst     w16,#ARMV8_SHA512
 218         b.ne    .Lv8_entry
 219 ___
 220 $code.=<<___;
 221 #endif
 222         AARCH64_SIGN_LINK_REGISTER
 223         stp     x29,x30,[sp,#-128]!
 224         add     x29,sp,#0
 225
 226         stp     x19,x20,[sp,#16]
 227         stp     x21,x22,[sp,#32]
 228         stp     x23,x24,[sp,#48]
 229         stp     x25,x26,[sp,#64]
 230         stp     x27,x28,[sp,#80]
 231         sub     sp,sp,#4*$SZ
 232
 233         ldp     $A,$B,[$ctx]                            // load context
 234         ldp     $C,$D,[$ctx,#2*$SZ]
 235         ldp     $E,$F,[$ctx,#4*$SZ]
 236         add     $num,$inp,$num,lsl#`log(16*$SZ)/log(2)` // end of input
 237         ldp     $G,$H,[$ctx,#6*$SZ]
 238         adr     $Ktbl,.LK$BITS
 239         stp     $ctx,$num,[x29,#96]
 240
 241 .Loop:
 242         ldp     @X[0],@X[1],[$inp],#2*$SZ
 243         ldr     $t2,[$Ktbl],#$SZ                        // *K++
 244         eor     $t3,$B,$C                               // magic seed
 245         str     $inp,[x29,#112]
 246 ___
 247 for ($i=0;$i<16;$i++)   { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
 248 $code.=".Loop_16_xx:\n";
 249 for (;$i<32;$i++)       { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
 250 $code.=<<___;
 251         cbnz    $t2,.Loop_16_xx
 252
 253         ldp     $ctx,$num,[x29,#96]
 254         ldr     $inp,[x29,#112]
 255         sub     $Ktbl,$Ktbl,#`$SZ*($rounds+1)`          // rewind
 256
 257         ldp     @X[0],@X[1],[$ctx]
 258         ldp     @X[2],@X[3],[$ctx,#2*$SZ]
 259         add     $inp,$inp,#14*$SZ                       // advance input pointer
 260         ldp     @X[4],@X[5],[$ctx,#4*$SZ]
 261         add     $A,$A,@X[0]
 262         ldp     @X[6],@X[7],[$ctx,#6*$SZ]
 263         add     $B,$B,@X[1]
 264         add     $C,$C,@X[2]
 265         add     $D,$D,@X[3]
 266         stp     $A,$B,[$ctx]
 267         add     $E,$E,@X[4]
 268         add     $F,$F,@X[5]
 269         stp     $C,$D,[$ctx,#2*$SZ]
 270         add     $G,$G,@X[6]
 271         add     $H,$H,@X[7]
 272         cmp     $inp,$num
 273         stp     $E,$F,[$ctx,#4*$SZ]
 274         stp     $G,$H,[$ctx,#6*$SZ]
 275         b.ne    .Loop
 276
 277         ldp     x19,x20,[x29,#16]
 278         add     sp,sp,#4*$SZ
 279         ldp     x21,x22,[x29,#32]
 280         ldp     x23,x24,[x29,#48]
 281         ldp     x25,x26,[x29,#64]
 282         ldp     x27,x28,[x29,#80]
 283         ldp     x29,x30,[sp],#128
 284         AARCH64_VALIDATE_LINK_REGISTER
 285         ret
 286 .size   $func,.-$func
 287
 288 .align  6
 289 .type   .LK$BITS,%object
 290 .LK$BITS:
 291 ___
 292 $code.=<<___ if ($SZ==8);
 293         .quad   0x428a2f98d728ae22,0x7137449123ef65cd
 294         .quad   0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
 295         .quad   0x3956c25bf348b538,0x59f111f1b605d019
 296         .quad   0x923f82a4af194f9b,0xab1c5ed5da6d8118
 297         .quad   0xd807aa98a3030242,0x12835b0145706fbe
 298         .quad   0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
 299         .quad   0x72be5d74f27b896f,0x80deb1fe3b1696b1
 300         .quad   0x9bdc06a725c71235,0xc19bf174cf692694
 301         .quad   0xe49b69c19ef14ad2,0xefbe4786384f25e3
 302         .quad   0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
 303         .quad   0x2de92c6f592b0275,0x4a7484aa6ea6e483
 304         .quad   0x5cb0a9dcbd41fbd4,0x76f988da831153b5
 305         .quad   0x983e5152ee66dfab,0xa831c66d2db43210
 306         .quad   0xb00327c898fb213f,0xbf597fc7beef0ee4
 307         .quad   0xc6e00bf33da88fc2,0xd5a79147930aa725
 308         .quad   0x06ca6351e003826f,0x142929670a0e6e70
 309         .quad   0x27b70a8546d22ffc,0x2e1b21385c26c926
 310         .quad   0x4d2c6dfc5ac42aed,0x53380d139d95b3df
 311         .quad   0x650a73548baf63de,0x766a0abb3c77b2a8
 312         .quad   0x81c2c92e47edaee6,0x92722c851482353b
 313         .quad   0xa2bfe8a14cf10364,0xa81a664bbc423001
 314         .quad   0xc24b8b70d0f89791,0xc76c51a30654be30
 315         .quad   0xd192e819d6ef5218,0xd69906245565a910
 316         .quad   0xf40e35855771202a,0x106aa07032bbd1b8
 317         .quad   0x19a4c116b8d2d0c8,0x1e376c085141ab53
 318         .quad   0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
 319         .quad   0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
 320         .quad   0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
 321         .quad   0x748f82ee5defb2fc,0x78a5636f43172f60
 322         .quad   0x84c87814a1f0ab72,0x8cc702081a6439ec
 323         .quad   0x90befffa23631e28,0xa4506cebde82bde9
 324         .quad   0xbef9a3f7b2c67915,0xc67178f2e372532b
 325         .quad   0xca273eceea26619c,0xd186b8c721c0c207
 326         .quad   0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
 327         .quad   0x06f067aa72176fba,0x0a637dc5a2c898a6
 328         .quad   0x113f9804bef90dae,0x1b710b35131c471b
 329         .quad   0x28db77f523047d84,0x32caab7b40c72493
 330         .quad   0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
 331         .quad   0x4cc5d4becb3e42b6,0x597f299cfc657e2a
 332         .quad   0x5fcb6fab3ad6faec,0x6c44198c4a475817
 333         .quad   0       // terminator
 334 ___
 335 $code.=<<___ if ($SZ==4);
 336         .long   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
 337         .long   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
 338         .long   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
 339         .long   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
 340         .long   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
 341         .long   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
 342         .long   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
 343         .long   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
 344         .long   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
 345         .long   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
 346         .long   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
 347         .long   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
 348         .long   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
 349         .long   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
 350         .long   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
 351         .long   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
 352         .long   0       //terminator
 353 ___
 354 $code.=<<___;
 355 .size   .LK$BITS,.-.LK$BITS
 356 .asciz  "SHA$BITS block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
 357 .align  2
 358 ___
 359
 360 if ($SZ==4) {
 361 my $Ktbl="x3";
 362
 363 my ($ABCD,$EFGH,$abcd)=map("v$_.16b",(0..2));
 364 my @MSG=map("v$_.16b",(4..7));
 365 my ($W0,$W1)=("v16.4s","v17.4s");
 366 my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
 367
 368 $code.=<<___;
 369 #ifndef __KERNEL__
 370 .type   sha256_block_armv8,%function
 371 .align  6
 372 sha256_block_armv8:
 373 .Lv8_entry:
 374         // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
 375         stp             x29,x30,[sp,#-16]!
 376         add             x29,sp,#0
 377
 378         ld1.32          {$ABCD,$EFGH},[$ctx]
 379         adr             $Ktbl,.LK256
 380
 381 .Loop_hw:
 382         ld1             {@MSG[0]-@MSG[3]},[$inp],#64
 383         sub             $num,$num,#1
 384         ld1.32          {$W0},[$Ktbl],#16
 385         rev32           @MSG[0],@MSG[0]
 386         rev32           @MSG[1],@MSG[1]
 387         rev32           @MSG[2],@MSG[2]
 388         rev32           @MSG[3],@MSG[3]
 389         orr             $ABCD_SAVE,$ABCD,$ABCD          // offload
 390         orr             $EFGH_SAVE,$EFGH,$EFGH
 391 ___
 392 for($i=0;$i<12;$i++) {
 393 $code.=<<___;
 394         ld1.32          {$W1},[$Ktbl],#16
 395         add.i32         $W0,$W0,@MSG[0]
 396         sha256su0       @MSG[0],@MSG[1]
 397         orr             $abcd,$ABCD,$ABCD
 398         sha256h         $ABCD,$EFGH,$W0
 399         sha256h2        $EFGH,$abcd,$W0
 400         sha256su1       @MSG[0],@MSG[2],@MSG[3]
 401 ___
 402         ($W0,$W1)=($W1,$W0);    push(@MSG,shift(@MSG));
 403 }
 404 $code.=<<___;
 405         ld1.32          {$W1},[$Ktbl],#16
 406         add.i32         $W0,$W0,@MSG[0]
 407         orr             $abcd,$ABCD,$ABCD
 408         sha256h         $ABCD,$EFGH,$W0
 409         sha256h2        $EFGH,$abcd,$W0
 410
 411         ld1.32          {$W0},[$Ktbl],#16
 412         add.i32         $W1,$W1,@MSG[1]
 413         orr             $abcd,$ABCD,$ABCD
 414         sha256h         $ABCD,$EFGH,$W1
 415         sha256h2        $EFGH,$abcd,$W1
 416
 417         ld1.32          {$W1},[$Ktbl]
 418         add.i32         $W0,$W0,@MSG[2]
 419         sub             $Ktbl,$Ktbl,#$rounds*$SZ-16     // rewind
 420         orr             $abcd,$ABCD,$ABCD
 421         sha256h         $ABCD,$EFGH,$W0
 422         sha256h2        $EFGH,$abcd,$W0
 423
 424         add.i32         $W1,$W1,@MSG[3]
 425         orr             $abcd,$ABCD,$ABCD
 426         sha256h         $ABCD,$EFGH,$W1
 427         sha256h2        $EFGH,$abcd,$W1
 428
 429         add.i32         $ABCD,$ABCD,$ABCD_SAVE
 430         add.i32         $EFGH,$EFGH,$EFGH_SAVE
 431
 432         cbnz            $num,.Loop_hw
 433
 434         st1.32          {$ABCD,$EFGH},[$ctx]
 435
 436         ldr             x29,[sp],#16
 437         ret
 438 .size   sha256_block_armv8,.-sha256_block_armv8
 439 #endif
 440 ___
 441 }
 442
 443 if ($SZ==4) {   ######################################### NEON stuff #
 444 # You'll surely note a lot of similarities with sha256-armv4 module,
 445 # and of course it's not a coincidence. sha256-armv4 was used as
 446 # initial template, but was adapted for ARMv8 instruction set and
 447 # extensively re-tuned for all-round performance.
 448
 449 my @V = ($A,$B,$C,$D,$E,$F,$G,$H) = map("w$_",(3..10));
 450 my ($t0,$t1,$t2,$t3,$t4) = map("w$_",(11..15));
 451 my $Ktbl="x16";
 452 my $Xfer="x17";
 453 my @X = map("q$_",(0..3));
 454 my ($T0,$T1,$T2,$T3,$T4,$T5,$T6,$T7) = map("q$_",(4..7,16..19));
 455 my $j=0;
 456
 457 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
 458 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
 459   my $arg = pop;
 460     $arg = "#$arg" if ($arg*1 eq $arg);
 461     $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
 462 }
 463
 464 sub Dscalar { shift =~ m|[qv]([0-9]+)|?"d$1":""; }
 465 sub Dlo     { shift =~ m|[qv]([0-9]+)|?"v$1.d[0]":""; }
 466 sub Dhi     { shift =~ m|[qv]([0-9]+)|?"v$1.d[1]":""; }
 467
 468 sub Xupdate()
 469 { use integer;
 470   my $body = shift;
 471   my @insns = (&$body,&$body,&$body,&$body);
 472   my ($a,$b,$c,$d,$e,$f,$g,$h);
 473
 474         &ext_8          ($T0,@X[0],@X[1],4);    # X[1..4]
 475          eval(shift(@insns));
 476          eval(shift(@insns));
 477          eval(shift(@insns));
 478         &ext_8          ($T3,@X[2],@X[3],4);    # X[9..12]
 479          eval(shift(@insns));
 480          eval(shift(@insns));
 481         &mov            (&Dscalar($T7),&Dhi(@X[3]));    # X[14..15]
 482          eval(shift(@insns));
 483          eval(shift(@insns));
 484         &ushr_32        ($T2,$T0,$sigma0[0]);
 485          eval(shift(@insns));
 486         &ushr_32        ($T1,$T0,$sigma0[2]);
 487          eval(shift(@insns));
 488         &add_32         (@X[0],@X[0],$T3);      # X[0..3] += X[9..12]
 489          eval(shift(@insns));
 490         &sli_32         ($T2,$T0,32-$sigma0[0]);
 491          eval(shift(@insns));
 492          eval(shift(@insns));
 493         &ushr_32        ($T3,$T0,$sigma0[1]);
 494          eval(shift(@insns));
 495          eval(shift(@insns));
 496         &eor_8          ($T1,$T1,$T2);
 497          eval(shift(@insns));
 498          eval(shift(@insns));
 499         &sli_32         ($T3,$T0,32-$sigma0[1]);
 500          eval(shift(@insns));
 501          eval(shift(@insns));
 502           &ushr_32      ($T4,$T7,$sigma1[0]);
 503          eval(shift(@insns));
 504          eval(shift(@insns));
 505         &eor_8          ($T1,$T1,$T3);          # sigma0(X[1..4])
 506          eval(shift(@insns));
 507          eval(shift(@insns));
 508           &sli_32       ($T4,$T7,32-$sigma1[0]);
 509          eval(shift(@insns));
 510          eval(shift(@insns));
 511           &ushr_32      ($T5,$T7,$sigma1[2]);
 512          eval(shift(@insns));
 513          eval(shift(@insns));
 514           &ushr_32      ($T3,$T7,$sigma1[1]);
 515          eval(shift(@insns));
 516          eval(shift(@insns));
 517         &add_32         (@X[0],@X[0],$T1);      # X[0..3] += sigma0(X[1..4])
 518          eval(shift(@insns));
 519          eval(shift(@insns));
 520           &sli_u32      ($T3,$T7,32-$sigma1[1]);
 521          eval(shift(@insns));
 522          eval(shift(@insns));
 523           &eor_8        ($T5,$T5,$T4);
 524          eval(shift(@insns));
 525          eval(shift(@insns));
 526          eval(shift(@insns));
 527           &eor_8        ($T5,$T5,$T3);          # sigma1(X[14..15])
 528          eval(shift(@insns));
 529          eval(shift(@insns));
 530          eval(shift(@insns));
 531         &add_32         (@X[0],@X[0],$T5);      # X[0..1] += sigma1(X[14..15])
 532          eval(shift(@insns));
 533          eval(shift(@insns));
 534          eval(shift(@insns));
 535           &ushr_32      ($T6,@X[0],$sigma1[0]);
 536          eval(shift(@insns));
 537           &ushr_32      ($T7,@X[0],$sigma1[2]);
 538          eval(shift(@insns));
 539          eval(shift(@insns));
 540           &sli_32       ($T6,@X[0],32-$sigma1[0]);
 541          eval(shift(@insns));
 542           &ushr_32      ($T5,@X[0],$sigma1[1]);
 543          eval(shift(@insns));
 544          eval(shift(@insns));
 545           &eor_8        ($T7,$T7,$T6);
 546          eval(shift(@insns));
 547          eval(shift(@insns));
 548           &sli_32       ($T5,@X[0],32-$sigma1[1]);
 549          eval(shift(@insns));
 550          eval(shift(@insns));
 551         &ld1_32         ("{$T0}","[$Ktbl], #16");
 552          eval(shift(@insns));
 553           &eor_8        ($T7,$T7,$T5);          # sigma1(X[16..17])
 554          eval(shift(@insns));
 555          eval(shift(@insns));
 556         &eor_8          ($T5,$T5,$T5);
 557          eval(shift(@insns));
 558          eval(shift(@insns));
 559         &mov            (&Dhi($T5), &Dlo($T7));
 560          eval(shift(@insns));
 561          eval(shift(@insns));
 562          eval(shift(@insns));
 563         &add_32         (@X[0],@X[0],$T5);      # X[2..3] += sigma1(X[16..17])
 564          eval(shift(@insns));
 565          eval(shift(@insns));
 566          eval(shift(@insns));
 567         &add_32         ($T0,$T0,@X[0]);
 568          while($#insns>=1) { eval(shift(@insns)); }
 569         &st1_32         ("{$T0}","[$Xfer], #16");
 570          eval(shift(@insns));
 571
 572         push(@X,shift(@X));             # "rotate" X[]
 573 }
 574
 575 sub Xpreload()
 576 { use integer;
 577   my $body = shift;
 578   my @insns = (&$body,&$body,&$body,&$body);
 579   my ($a,$b,$c,$d,$e,$f,$g,$h);
 580
 581          eval(shift(@insns));
 582          eval(shift(@insns));
 583         &ld1_8          ("{@X[0]}","[$inp],#16");
 584          eval(shift(@insns));
 585          eval(shift(@insns));
 586         &ld1_32         ("{$T0}","[$Ktbl],#16");
 587          eval(shift(@insns));
 588          eval(shift(@insns));
 589          eval(shift(@insns));
 590          eval(shift(@insns));
 591         &rev32          (@X[0],@X[0]);
 592          eval(shift(@insns));
 593          eval(shift(@insns));
 594          eval(shift(@insns));
 595          eval(shift(@insns));
 596         &add_32         ($T0,$T0,@X[0]);
 597          foreach (@insns) { eval; }     # remaining instructions
 598         &st1_32         ("{$T0}","[$Xfer], #16");
 599
 600         push(@X,shift(@X));             # "rotate" X[]
 601 }
 602
 603 sub body_00_15 () {
 604         (
 605         '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
 606         '&add   ($h,$h,$t1)',                   # h+=X[i]+K[i]
 607         '&add   ($a,$a,$t4);'.                  # h+=Sigma0(a) from the past
 608         '&and   ($t1,$f,$e)',
 609         '&bic   ($t4,$g,$e)',
 610         '&eor   ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
 611         '&add   ($a,$a,$t2)',                   # h+=Maj(a,b,c) from the past
 612         '&orr   ($t1,$t1,$t4)',                 # Ch(e,f,g)
 613         '&eor   ($t0,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))',   # Sigma1(e)
 614         '&eor   ($t4,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
 615         '&add   ($h,$h,$t1)',                   # h+=Ch(e,f,g)
 616         '&ror   ($t0,$t0,"#$Sigma1[0]")',
 617         '&eor   ($t2,$a,$b)',                   # a^b, b^c in next round
 618         '&eor   ($t4,$t4,$a,"ror#".($Sigma0[2]-$Sigma0[0]))',   # Sigma0(a)
 619         '&add   ($h,$h,$t0)',                   # h+=Sigma1(e)
 620         '&ldr   ($t1,sprintf "[sp,#%d]",4*(($j+1)&15))  if (($j&15)!=15);'.
 621         '&ldr   ($t1,"[$Ktbl]")                         if ($j==15);'.
 622         '&and   ($t3,$t3,$t2)',                 # (b^c)&=(a^b)
 623         '&ror   ($t4,$t4,"#$Sigma0[0]")',
 624         '&add   ($d,$d,$h)',                    # d+=h
 625         '&eor   ($t3,$t3,$b)',                  # Maj(a,b,c)
 626         '$j++;  unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
 627         )
 628 }
 629
 630 $code.=<<___;
 631 #ifdef  __KERNEL__
 632 .globl  sha256_block_neon
 633 #endif
 634 .type   sha256_block_neon,%function
 635 .align  4
 636 sha256_block_neon:
 637         AARCH64_VALID_CALL_TARGET
 638 .Lneon_entry:
 639         // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later
 640         stp     x29, x30, [sp, #-16]!
 641         mov     x29, sp
 642         sub     sp,sp,#16*4
 643
 644         adr     $Ktbl,.LK256
 645         add     $num,$inp,$num,lsl#6    // len to point at the end of inp
 646
 647         ld1.8   {@X[0]},[$inp], #16
 648         ld1.8   {@X[1]},[$inp], #16
 649         ld1.8   {@X[2]},[$inp], #16
 650         ld1.8   {@X[3]},[$inp], #16
 651         ld1.32  {$T0},[$Ktbl], #16
 652         ld1.32  {$T1},[$Ktbl], #16
 653         ld1.32  {$T2},[$Ktbl], #16
 654         ld1.32  {$T3},[$Ktbl], #16
 655         rev32   @X[0],@X[0]             // yes, even on
 656         rev32   @X[1],@X[1]             // big-endian
 657         rev32   @X[2],@X[2]
 658         rev32   @X[3],@X[3]
 659         mov     $Xfer,sp
 660         add.32  $T0,$T0,@X[0]
 661         add.32  $T1,$T1,@X[1]
 662         add.32  $T2,$T2,@X[2]
 663         st1.32  {$T0-$T1},[$Xfer], #32
 664         add.32  $T3,$T3,@X[3]
 665         st1.32  {$T2-$T3},[$Xfer]
 666         sub     $Xfer,$Xfer,#32
 667
 668         ldp     $A,$B,[$ctx]
 669         ldp     $C,$D,[$ctx,#8]
 670         ldp     $E,$F,[$ctx,#16]
 671         ldp     $G,$H,[$ctx,#24]
 672         ldr     $t1,[sp,#0]
 673         mov     $t2,wzr
 674         eor     $t3,$B,$C
 675         mov     $t4,wzr
 676         b       .L_00_48
 677
 678 .align  4
 679 .L_00_48:
 680 ___
 681         &Xupdate(\&body_00_15);
 682         &Xupdate(\&body_00_15);
 683         &Xupdate(\&body_00_15);
 684         &Xupdate(\&body_00_15);
 685 $code.=<<___;
 686         cmp     $t1,#0                          // check for K256 terminator
 687         ldr     $t1,[sp,#0]
 688         sub     $Xfer,$Xfer,#64
 689         bne     .L_00_48
 690
 691         sub     $Ktbl,$Ktbl,#256                // rewind $Ktbl
 692         cmp     $inp,$num
 693         mov     $Xfer, #64
 694         csel    $Xfer, $Xfer, xzr, eq
 695         sub     $inp,$inp,$Xfer                 // avoid SEGV
 696         mov     $Xfer,sp
 697 ___
 698         &Xpreload(\&body_00_15);
 699         &Xpreload(\&body_00_15);
 700         &Xpreload(\&body_00_15);
 701         &Xpreload(\&body_00_15);
 702 $code.=<<___;
 703         add     $A,$A,$t4                       // h+=Sigma0(a) from the past
 704         ldp     $t0,$t1,[$ctx,#0]
 705         add     $A,$A,$t2                       // h+=Maj(a,b,c) from the past
 706         ldp     $t2,$t3,[$ctx,#8]
 707         add     $A,$A,$t0                       // accumulate
 708         add     $B,$B,$t1
 709         ldp     $t0,$t1,[$ctx,#16]
 710         add     $C,$C,$t2
 711         add     $D,$D,$t3
 712         ldp     $t2,$t3,[$ctx,#24]
 713         add     $E,$E,$t0
 714         add     $F,$F,$t1
 715          ldr    $t1,[sp,#0]
 716         stp     $A,$B,[$ctx,#0]
 717         add     $G,$G,$t2
 718          mov    $t2,wzr
 719         stp     $C,$D,[$ctx,#8]
 720         add     $H,$H,$t3
 721         stp     $E,$F,[$ctx,#16]
 722          eor    $t3,$B,$C
 723         stp     $G,$H,[$ctx,#24]
 724          mov    $t4,wzr
 725          mov    $Xfer,sp
 726         b.ne    .L_00_48
 727
 728         ldr     x29,[x29]
 729         add     sp,sp,#16*4+16
 730         ret
 731 .size   sha256_block_neon,.-sha256_block_neon
 732 ___
 733 }
 734
 735 if ($SZ==8) {
 736 my $Ktbl="x3";
 737
 738 my @H = map("v$_.16b",(0..4));
 739 my ($fg,$de,$m9_10)=map("v$_.16b",(5..7));
 740 my @MSG=map("v$_.16b",(16..23));
 741 my ($W0,$W1)=("v24.2d","v25.2d");
 742 my ($AB,$CD,$EF,$GH)=map("v$_.16b",(26..29));
 743
 744 $code.=<<___;
 745 #ifndef __KERNEL__
 746 .type   sha512_block_armv8,%function
 747 .align  6
 748 sha512_block_armv8:
 749 .Lv8_entry:
 750         // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later
 751         stp             x29,x30,[sp,#-16]!
 752         add             x29,sp,#0
 753
 754         ld1             {@MSG[0]-@MSG[3]},[$inp],#64    // load input
 755         ld1             {@MSG[4]-@MSG[7]},[$inp],#64
 756
 757         ld1.64          {@H[0]-@H[3]},[$ctx]            // load context
 758         adr             $Ktbl,.LK512
 759
 760         rev64           @MSG[0],@MSG[0]
 761         rev64           @MSG[1],@MSG[1]
 762         rev64           @MSG[2],@MSG[2]
 763         rev64           @MSG[3],@MSG[3]
 764         rev64           @MSG[4],@MSG[4]
 765         rev64           @MSG[5],@MSG[5]
 766         rev64           @MSG[6],@MSG[6]
 767         rev64           @MSG[7],@MSG[7]
 768         b               .Loop_hw
 769
 770 .align  4
 771 .Loop_hw:
 772         ld1.64          {$W0},[$Ktbl],#16
 773         subs            $num,$num,#1
 774         sub             x4,$inp,#128
 775         orr             $AB,@H[0],@H[0]                 // offload
 776         orr             $CD,@H[1],@H[1]
 777         orr             $EF,@H[2],@H[2]
 778         orr             $GH,@H[3],@H[3]
 779         csel            $inp,$inp,x4,ne                 // conditional rewind
 780 ___
 781 for($i=0;$i<32;$i++) {
 782 $code.=<<___;
 783         add.i64         $W0,$W0,@MSG[0]
 784         ld1.64          {$W1},[$Ktbl],#16
 785         ext             $W0,$W0,$W0,#8
 786         ext             $fg,@H[2],@H[3],#8
 787         ext             $de,@H[1],@H[2],#8
 788         add.i64         @H[3],@H[3],$W0                 // "T1 + H + K512[i]"
 789          sha512su0      @MSG[0],@MSG[1]
 790          ext            $m9_10,@MSG[4],@MSG[5],#8
 791         sha512h         @H[3],$fg,$de
 792          sha512su1      @MSG[0],@MSG[7],$m9_10
 793         add.i64         @H[4],@H[1],@H[3]               // "D + T1"
 794         sha512h2        @H[3],$H[1],@H[0]
 795 ___
 796         ($W0,$W1)=($W1,$W0);    push(@MSG,shift(@MSG));
 797         @H = (@H[3],@H[0],@H[4],@H[2],@H[1]);
 798 }
 799 for(;$i<40;$i++) {
 800 $code.=<<___    if ($i<39);
 801         ld1.64          {$W1},[$Ktbl],#16
 802 ___
 803 $code.=<<___    if ($i==39);
 804         sub             $Ktbl,$Ktbl,#$rounds*$SZ        // rewind
 805 ___
 806 $code.=<<___;
 807         add.i64         $W0,$W0,@MSG[0]
 808          ld1            {@MSG[0]},[$inp],#16            // load next input
 809         ext             $W0,$W0,$W0,#8
 810         ext             $fg,@H[2],@H[3],#8
 811         ext             $de,@H[1],@H[2],#8
 812         add.i64         @H[3],@H[3],$W0                 // "T1 + H + K512[i]"
 813         sha512h         @H[3],$fg,$de
 814          rev64          @MSG[0],@MSG[0]
 815         add.i64         @H[4],@H[1],@H[3]               // "D + T1"
 816         sha512h2        @H[3],$H[1],@H[0]
 817 ___
 818         ($W0,$W1)=($W1,$W0);    push(@MSG,shift(@MSG));
 819         @H = (@H[3],@H[0],@H[4],@H[2],@H[1]);
 820 }
 821 $code.=<<___;
 822         add.i64         @H[0],@H[0],$AB                 // accumulate
 823         add.i64         @H[1],@H[1],$CD
 824         add.i64         @H[2],@H[2],$EF
 825         add.i64         @H[3],@H[3],$GH
 826
 827         cbnz            $num,.Loop_hw
 828
 829         st1.64          {@H[0]-@H[3]},[$ctx]            // store context
 830
 831         ldr             x29,[sp],#16
 832         ret
 833 .size   sha512_block_armv8,.-sha512_block_armv8
 834 #endif
 835 ___
 836 }
 837
 838 {   my  %opcode = (
 839         "sha256h"       => 0x5e004000,  "sha256h2"      => 0x5e005000,
 840         "sha256su0"     => 0x5e282800,  "sha256su1"     => 0x5e006000   );
 841
 842     sub unsha256 {
 843         my ($mnemonic,$arg)=@_;
 844
 845         $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
 846         &&
 847         sprintf ".inst\t0x%08x\t//%s %s",
 848                         $opcode{$mnemonic}|$1|($2<<5)|($3<<16),
 849                         $mnemonic,$arg;
 850     }
 851 }
 852
 853 {   my  %opcode = (
 854         "sha512h"       => 0xce608000,  "sha512h2"      => 0xce608400,
 855         "sha512su0"     => 0xcec08000,  "sha512su1"     => 0xce608800   );
 856
 857     sub unsha512 {
 858         my ($mnemonic,$arg)=@_;
 859
 860         $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
 861         &&
 862         sprintf ".inst\t0x%08x\t//%s %s",
 863                         $opcode{$mnemonic}|$1|($2<<5)|($3<<16),
 864                         $mnemonic,$arg;
 865     }
 866 }
 867
 868 open SELF,$0;
 869 while(<SELF>) {
 870         next if (/^#!/);
 871         last if (!s/^#/\/\// and !/^$/);
 872         print;
 873 }
 874 close SELF;
 875
 876 foreach(split("\n",$code)) {
 877
 878         s/\`([^\`]*)\`/eval($1)/ge;
 879
 880         s/\b(sha512\w+)\s+([qv].*)/unsha512($1,$2)/ge   or
 881         s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/ge;
 882
 883         s/\bq([0-9]+)\b/v$1.16b/g;              # old->new registers
 884
 885         s/\.[ui]?8(\s)/$1/;
 886         s/\.\w?64\b//           and s/\.16b/\.2d/g      or
 887         s/\.\w?32\b//           and s/\.16b/\.4s/g;
 888         m/\bext\b/              and s/\.2d/\.16b/g      or
 889         m/(ld|st)1[^\[]+\[0\]/  and s/\.4s/\.s/g;
 890
 891         print $_,"\n";
 892 }
 893
 894 close STDOUT or die "error closing STDOUT: $!";