crypto/sha/asm/sha512-armv8.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2014-2018 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License 2.0 (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9 # ====================================================================
  10 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  11 # project. The module is, however, dual licensed under OpenSSL and
  12 # CRYPTOGAMS licenses depending on where you obtain it. For further
  13 # details see http://www.openssl.org/~appro/cryptogams/.
  14 #
  15 # Permission to use under GPLv2 terms is granted.
  16 # ====================================================================
  17 #
  18 # SHA256/512 for ARMv8.
  19 #
  20 # Performance in cycles per processed byte and improvement coefficient
  21 # over code generated with "default" compiler:
  22 #
  23 #               SHA256-hw       SHA256(*)       SHA512
  24 # Apple A7      1.97            10.5 (+33%)     6.73 (-1%(**))
  25 # Cortex-A53    2.38            15.5 (+115%)    10.0 (+150%(***))
  26 # Cortex-A57    2.31            11.6 (+86%)     7.51 (+260%(***))
  27 # Denver        2.01            10.5 (+26%)     6.70 (+8%)
  28 # X-Gene                        20.0 (+100%)    12.8 (+300%(***))
  29 # Mongoose      2.36            13.0 (+50%)     8.36 (+33%)
  30 # Kryo          1.92            17.4 (+30%)     11.2 (+8%)
  31 # ThunderX2     2.54            13.2 (+40%)     8.40 (+18%)
  32 #
  33 # (*)   Software SHA256 results are of lesser relevance, presented
  34 #       mostly for informational purposes.
  35 # (**)  The result is a trade-off: it's possible to improve it by
  36 #       10% (or by 1 cycle per round), but at the cost of 20% loss
  37 #       on Cortex-A53 (or by 4 cycles per round).
  38 # (***) Super-impressive coefficients over gcc-generated code are
  39 #       indication of some compiler "pathology", most notably code
  40 #       generated with -mgeneral-regs-only is significantly faster
  41 #       and the gap is only 40-90%.
  42 #
  43 # October 2016.
  44 #
  45 # Originally it was reckoned that it makes no sense to implement NEON
  46 # version of SHA256 for 64-bit processors. This is because performance
  47 # improvement on most wide-spread Cortex-A5x processors was observed
  48 # to be marginal, same on Cortex-A53 and ~10% on A57. But then it was
  49 # observed that 32-bit NEON SHA256 performs significantly better than
  50 # 64-bit scalar version on *some* of the more recent processors. As
  51 # result 64-bit NEON version of SHA256 was added to provide best
  52 # all-round performance. For example it executes ~30% faster on X-Gene
  53 # and Mongoose. [For reference, NEON version of SHA512 is bound to
  54 # deliver much less improvement, likely *negative* on Cortex-A5x.
  55 # Which is why NEON support is limited to SHA256.]
  56
  57 # $output is the last argument if it looks like a file (it has an extension)
  58 # $flavour is the first argument if it doesn't look like a file
  59 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  60 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  61
  62 if ($flavour && $flavour ne "void") {
  63     $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  64     ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  65     ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  66     die "can't locate arm-xlate.pl";
  67
  68     open OUT,"| \"$^X\" $xlate $flavour \"$output\""
  69         or die "can't call $xlate: $!";
  70     *STDOUT=*OUT;
  71 } else {
  72     $output and open STDOUT,">$output";
  73 }
  74
  75 if ($output =~ /512/) {
  76         $BITS=512;
  77         $SZ=8;
  78         @Sigma0=(28,34,39);
  79         @Sigma1=(14,18,41);
  80         @sigma0=(1,  8, 7);
  81         @sigma1=(19,61, 6);
  82         $rounds=80;
  83         $reg_t="x";
  84 } else {
  85         $BITS=256;
  86         $SZ=4;
  87         @Sigma0=( 2,13,22);
  88         @Sigma1=( 6,11,25);
  89         @sigma0=( 7,18, 3);
  90         @sigma1=(17,19,10);
  91         $rounds=64;
  92         $reg_t="w";
  93 }
  94
  95 $func="sha${BITS}_block_data_order";
  96
  97 ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
  98
  99 @X=map("$reg_t$_",(3..15,0..2));
 100 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27));
 101 ($t0,$t1,$t2,$t3)=map("$reg_t$_",(16,17,19,28));
 102
 103 sub BODY_00_xx {
 104 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
 105 my $j=($i+1)&15;
 106 my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]);
 107    $T0=@X[$i+3] if ($i<11);
 108
 109 $code.=<<___    if ($i<16);
 110 #ifndef __AARCH64EB__
 111         rev     @X[$i],@X[$i]                   // $i
 112 #endif
 113 ___
 114 $code.=<<___    if ($i<13 && ($i&1));
 115         ldp     @X[$i+1],@X[$i+2],[$inp],#2*$SZ
 116 ___
 117 $code.=<<___    if ($i==13);
 118         ldp     @X[14],@X[15],[$inp]
 119 ___
 120 $code.=<<___    if ($i>=14);
 121         ldr     @X[($i-11)&15],[sp,#`$SZ*(($i-11)%4)`]
 122 ___
 123 $code.=<<___    if ($i>0 && $i<16);
 124         add     $a,$a,$t1                       // h+=Sigma0(a)
 125 ___
 126 $code.=<<___    if ($i>=11);
 127         str     @X[($i-8)&15],[sp,#`$SZ*(($i-8)%4)`]
 128 ___
 129 # While ARMv8 specifies merged rotate-n-logical operation such as
 130 # 'eor x,y,z,ror#n', it was found to negatively affect performance
 131 # on Apple A7. The reason seems to be that it requires even 'y' to
 132 # be available earlier. This means that such merged instruction is
 133 # not necessarily best choice on critical path... On the other hand
 134 # Cortex-A5x handles merged instructions much better than disjoint
 135 # rotate and logical... See (**) footnote above.
 136 $code.=<<___    if ($i<15);
 137         ror     $t0,$e,#$Sigma1[0]
 138         add     $h,$h,$t2                       // h+=K[i]
 139         eor     $T0,$e,$e,ror#`$Sigma1[2]-$Sigma1[1]`
 140         and     $t1,$f,$e
 141         bic     $t2,$g,$e
 142         add     $h,$h,@X[$i&15]                 // h+=X[i]
 143         orr     $t1,$t1,$t2                     // Ch(e,f,g)
 144         eor     $t2,$a,$b                       // a^b, b^c in next round
 145         eor     $t0,$t0,$T0,ror#$Sigma1[1]      // Sigma1(e)
 146         ror     $T0,$a,#$Sigma0[0]
 147         add     $h,$h,$t1                       // h+=Ch(e,f,g)
 148         eor     $t1,$a,$a,ror#`$Sigma0[2]-$Sigma0[1]`
 149         add     $h,$h,$t0                       // h+=Sigma1(e)
 150         and     $t3,$t3,$t2                     // (b^c)&=(a^b)
 151         add     $d,$d,$h                        // d+=h
 152         eor     $t3,$t3,$b                      // Maj(a,b,c)
 153         eor     $t1,$T0,$t1,ror#$Sigma0[1]      // Sigma0(a)
 154         add     $h,$h,$t3                       // h+=Maj(a,b,c)
 155         ldr     $t3,[$Ktbl],#$SZ                // *K++, $t2 in next round
 156         //add   $h,$h,$t1                       // h+=Sigma0(a)
 157 ___
 158 $code.=<<___    if ($i>=15);
 159         ror     $t0,$e,#$Sigma1[0]
 160         add     $h,$h,$t2                       // h+=K[i]
 161         ror     $T1,@X[($j+1)&15],#$sigma0[0]
 162         and     $t1,$f,$e
 163         ror     $T2,@X[($j+14)&15],#$sigma1[0]
 164         bic     $t2,$g,$e
 165         ror     $T0,$a,#$Sigma0[0]
 166         add     $h,$h,@X[$i&15]                 // h+=X[i]
 167         eor     $t0,$t0,$e,ror#$Sigma1[1]
 168         eor     $T1,$T1,@X[($j+1)&15],ror#$sigma0[1]
 169         orr     $t1,$t1,$t2                     // Ch(e,f,g)
 170         eor     $t2,$a,$b                       // a^b, b^c in next round
 171         eor     $t0,$t0,$e,ror#$Sigma1[2]       // Sigma1(e)
 172         eor     $T0,$T0,$a,ror#$Sigma0[1]
 173         add     $h,$h,$t1                       // h+=Ch(e,f,g)
 174         and     $t3,$t3,$t2                     // (b^c)&=(a^b)
 175         eor     $T2,$T2,@X[($j+14)&15],ror#$sigma1[1]
 176         eor     $T1,$T1,@X[($j+1)&15],lsr#$sigma0[2]    // sigma0(X[i+1])
 177         add     $h,$h,$t0                       // h+=Sigma1(e)
 178         eor     $t3,$t3,$b                      // Maj(a,b,c)
 179         eor     $t1,$T0,$a,ror#$Sigma0[2]       // Sigma0(a)
 180         eor     $T2,$T2,@X[($j+14)&15],lsr#$sigma1[2]   // sigma1(X[i+14])
 181         add     @X[$j],@X[$j],@X[($j+9)&15]
 182         add     $d,$d,$h                        // d+=h
 183         add     $h,$h,$t3                       // h+=Maj(a,b,c)
 184         ldr     $t3,[$Ktbl],#$SZ                // *K++, $t2 in next round
 185         add     @X[$j],@X[$j],$T1
 186         add     $h,$h,$t1                       // h+=Sigma0(a)
 187         add     @X[$j],@X[$j],$T2
 188 ___
 189         ($t2,$t3)=($t3,$t2);
 190 }
 191
 192 $code.=<<___;
 193 #ifndef __KERNEL__
 194 # include "arm_arch.h"
 195 .extern OPENSSL_armcap_P
 196 #endif
 197
 198 .text
 199
 200 .globl  $func
 201 .type   $func,%function
 202 .align  6
 203 $func:
 204 #ifndef __KERNEL__
 205         adrp    x16,OPENSSL_armcap_P
 206         ldr     w16,[x16,#:lo12:OPENSSL_armcap_P]
 207 ___
 208 $code.=<<___    if ($SZ==4);
 209         tst     w16,#ARMV8_SHA256
 210         b.ne    .Lv8_entry
 211         tst     w16,#ARMV7_NEON
 212         b.ne    .Lneon_entry
 213 ___
 214 $code.=<<___    if ($SZ==8);
 215         tst     w16,#ARMV8_SHA512
 216         b.ne    .Lv8_entry
 217 ___
 218 $code.=<<___;
 219 #endif
 220         .inst   0xd503233f                              // paciasp
 221         stp     x29,x30,[sp,#-128]!
 222         add     x29,sp,#0
 223
 224         stp     x19,x20,[sp,#16]
 225         stp     x21,x22,[sp,#32]
 226         stp     x23,x24,[sp,#48]
 227         stp     x25,x26,[sp,#64]
 228         stp     x27,x28,[sp,#80]
 229         sub     sp,sp,#4*$SZ
 230
 231         ldp     $A,$B,[$ctx]                            // load context
 232         ldp     $C,$D,[$ctx,#2*$SZ]
 233         ldp     $E,$F,[$ctx,#4*$SZ]
 234         add     $num,$inp,$num,lsl#`log(16*$SZ)/log(2)` // end of input
 235         ldp     $G,$H,[$ctx,#6*$SZ]
 236         adr     $Ktbl,.LK$BITS
 237         stp     $ctx,$num,[x29,#96]
 238
 239 .Loop:
 240         ldp     @X[0],@X[1],[$inp],#2*$SZ
 241         ldr     $t2,[$Ktbl],#$SZ                        // *K++
 242         eor     $t3,$B,$C                               // magic seed
 243         str     $inp,[x29,#112]
 244 ___
 245 for ($i=0;$i<16;$i++)   { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
 246 $code.=".Loop_16_xx:\n";
 247 for (;$i<32;$i++)       { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
 248 $code.=<<___;
 249         cbnz    $t2,.Loop_16_xx
 250
 251         ldp     $ctx,$num,[x29,#96]
 252         ldr     $inp,[x29,#112]
 253         sub     $Ktbl,$Ktbl,#`$SZ*($rounds+1)`          // rewind
 254
 255         ldp     @X[0],@X[1],[$ctx]
 256         ldp     @X[2],@X[3],[$ctx,#2*$SZ]
 257         add     $inp,$inp,#14*$SZ                       // advance input pointer
 258         ldp     @X[4],@X[5],[$ctx,#4*$SZ]
 259         add     $A,$A,@X[0]
 260         ldp     @X[6],@X[7],[$ctx,#6*$SZ]
 261         add     $B,$B,@X[1]
 262         add     $C,$C,@X[2]
 263         add     $D,$D,@X[3]
 264         stp     $A,$B,[$ctx]
 265         add     $E,$E,@X[4]
 266         add     $F,$F,@X[5]
 267         stp     $C,$D,[$ctx,#2*$SZ]
 268         add     $G,$G,@X[6]
 269         add     $H,$H,@X[7]
 270         cmp     $inp,$num
 271         stp     $E,$F,[$ctx,#4*$SZ]
 272         stp     $G,$H,[$ctx,#6*$SZ]
 273         b.ne    .Loop
 274
 275         ldp     x19,x20,[x29,#16]
 276         add     sp,sp,#4*$SZ
 277         ldp     x21,x22,[x29,#32]
 278         ldp     x23,x24,[x29,#48]
 279         ldp     x25,x26,[x29,#64]
 280         ldp     x27,x28,[x29,#80]
 281         ldp     x29,x30,[sp],#128
 282         .inst   0xd50323bf                              // autiasp
 283         ret
 284 .size   $func,.-$func
 285
 286 .align  6
 287 .type   .LK$BITS,%object
 288 .LK$BITS:
 289 ___
 290 $code.=<<___ if ($SZ==8);
 291         .quad   0x428a2f98d728ae22,0x7137449123ef65cd
 292         .quad   0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
 293         .quad   0x3956c25bf348b538,0x59f111f1b605d019
 294         .quad   0x923f82a4af194f9b,0xab1c5ed5da6d8118
 295         .quad   0xd807aa98a3030242,0x12835b0145706fbe
 296         .quad   0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
 297         .quad   0x72be5d74f27b896f,0x80deb1fe3b1696b1
 298         .quad   0x9bdc06a725c71235,0xc19bf174cf692694
 299         .quad   0xe49b69c19ef14ad2,0xefbe4786384f25e3
 300         .quad   0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
 301         .quad   0x2de92c6f592b0275,0x4a7484aa6ea6e483
 302         .quad   0x5cb0a9dcbd41fbd4,0x76f988da831153b5
 303         .quad   0x983e5152ee66dfab,0xa831c66d2db43210
 304         .quad   0xb00327c898fb213f,0xbf597fc7beef0ee4
 305         .quad   0xc6e00bf33da88fc2,0xd5a79147930aa725
 306         .quad   0x06ca6351e003826f,0x142929670a0e6e70
 307         .quad   0x27b70a8546d22ffc,0x2e1b21385c26c926
 308         .quad   0x4d2c6dfc5ac42aed,0x53380d139d95b3df
 309         .quad   0x650a73548baf63de,0x766a0abb3c77b2a8
 310         .quad   0x81c2c92e47edaee6,0x92722c851482353b
 311         .quad   0xa2bfe8a14cf10364,0xa81a664bbc423001
 312         .quad   0xc24b8b70d0f89791,0xc76c51a30654be30
 313         .quad   0xd192e819d6ef5218,0xd69906245565a910
 314         .quad   0xf40e35855771202a,0x106aa07032bbd1b8
 315         .quad   0x19a4c116b8d2d0c8,0x1e376c085141ab53
 316         .quad   0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
 317         .quad   0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
 318         .quad   0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
 319         .quad   0x748f82ee5defb2fc,0x78a5636f43172f60
 320         .quad   0x84c87814a1f0ab72,0x8cc702081a6439ec
 321         .quad   0x90befffa23631e28,0xa4506cebde82bde9
 322         .quad   0xbef9a3f7b2c67915,0xc67178f2e372532b
 323         .quad   0xca273eceea26619c,0xd186b8c721c0c207
 324         .quad   0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
 325         .quad   0x06f067aa72176fba,0x0a637dc5a2c898a6
 326         .quad   0x113f9804bef90dae,0x1b710b35131c471b
 327         .quad   0x28db77f523047d84,0x32caab7b40c72493
 328         .quad   0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
 329         .quad   0x4cc5d4becb3e42b6,0x597f299cfc657e2a
 330         .quad   0x5fcb6fab3ad6faec,0x6c44198c4a475817
 331         .quad   0       // terminator
 332 ___
 333 $code.=<<___ if ($SZ==4);
 334         .long   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
 335         .long   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
 336         .long   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
 337         .long   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
 338         .long   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
 339         .long   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
 340         .long   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
 341         .long   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
 342         .long   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
 343         .long   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
 344         .long   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
 345         .long   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
 346         .long   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
 347         .long   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
 348         .long   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
 349         .long   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
 350         .long   0       //terminator
 351 ___
 352 $code.=<<___;
 353 .size   .LK$BITS,.-.LK$BITS
 354 .asciz  "SHA$BITS block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
 355 .align  2
 356 ___
 357
 358 if ($SZ==4) {
 359 my $Ktbl="x3";
 360
 361 my ($ABCD,$EFGH,$abcd)=map("v$_.16b",(0..2));
 362 my @MSG=map("v$_.16b",(4..7));
 363 my ($W0,$W1)=("v16.4s","v17.4s");
 364 my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
 365
 366 $code.=<<___;
 367 #ifndef __KERNEL__
 368 .type   sha256_block_armv8,%function
 369 .align  6
 370 sha256_block_armv8:
 371 .Lv8_entry:
 372         stp             x29,x30,[sp,#-16]!
 373         add             x29,sp,#0
 374
 375         ld1.32          {$ABCD,$EFGH},[$ctx]
 376         adr             $Ktbl,.LK256
 377
 378 .Loop_hw:
 379         ld1             {@MSG[0]-@MSG[3]},[$inp],#64
 380         sub             $num,$num,#1
 381         ld1.32          {$W0},[$Ktbl],#16
 382         rev32           @MSG[0],@MSG[0]
 383         rev32           @MSG[1],@MSG[1]
 384         rev32           @MSG[2],@MSG[2]
 385         rev32           @MSG[3],@MSG[3]
 386         orr             $ABCD_SAVE,$ABCD,$ABCD          // offload
 387         orr             $EFGH_SAVE,$EFGH,$EFGH
 388 ___
 389 for($i=0;$i<12;$i++) {
 390 $code.=<<___;
 391         ld1.32          {$W1},[$Ktbl],#16
 392         add.i32         $W0,$W0,@MSG[0]
 393         sha256su0       @MSG[0],@MSG[1]
 394         orr             $abcd,$ABCD,$ABCD
 395         sha256h         $ABCD,$EFGH,$W0
 396         sha256h2        $EFGH,$abcd,$W0
 397         sha256su1       @MSG[0],@MSG[2],@MSG[3]
 398 ___
 399         ($W0,$W1)=($W1,$W0);    push(@MSG,shift(@MSG));
 400 }
 401 $code.=<<___;
 402         ld1.32          {$W1},[$Ktbl],#16
 403         add.i32         $W0,$W0,@MSG[0]
 404         orr             $abcd,$ABCD,$ABCD
 405         sha256h         $ABCD,$EFGH,$W0
 406         sha256h2        $EFGH,$abcd,$W0
 407
 408         ld1.32          {$W0},[$Ktbl],#16
 409         add.i32         $W1,$W1,@MSG[1]
 410         orr             $abcd,$ABCD,$ABCD
 411         sha256h         $ABCD,$EFGH,$W1
 412         sha256h2        $EFGH,$abcd,$W1
 413
 414         ld1.32          {$W1},[$Ktbl]
 415         add.i32         $W0,$W0,@MSG[2]
 416         sub             $Ktbl,$Ktbl,#$rounds*$SZ-16     // rewind
 417         orr             $abcd,$ABCD,$ABCD
 418         sha256h         $ABCD,$EFGH,$W0
 419         sha256h2        $EFGH,$abcd,$W0
 420
 421         add.i32         $W1,$W1,@MSG[3]
 422         orr             $abcd,$ABCD,$ABCD
 423         sha256h         $ABCD,$EFGH,$W1
 424         sha256h2        $EFGH,$abcd,$W1
 425
 426         add.i32         $ABCD,$ABCD,$ABCD_SAVE
 427         add.i32         $EFGH,$EFGH,$EFGH_SAVE
 428
 429         cbnz            $num,.Loop_hw
 430
 431         st1.32          {$ABCD,$EFGH},[$ctx]
 432
 433         ldr             x29,[sp],#16
 434         ret
 435 .size   sha256_block_armv8,.-sha256_block_armv8
 436 #endif
 437 ___
 438 }
 439
 440 if ($SZ==4) {   ######################################### NEON stuff #
 441 # You'll surely note a lot of similarities with sha256-armv4 module,
 442 # and of course it's not a coincidence. sha256-armv4 was used as
 443 # initial template, but was adapted for ARMv8 instruction set and
 444 # extensively re-tuned for all-round performance.
 445
 446 my @V = ($A,$B,$C,$D,$E,$F,$G,$H) = map("w$_",(3..10));
 447 my ($t0,$t1,$t2,$t3,$t4) = map("w$_",(11..15));
 448 my $Ktbl="x16";
 449 my $Xfer="x17";
 450 my @X = map("q$_",(0..3));
 451 my ($T0,$T1,$T2,$T3,$T4,$T5,$T6,$T7) = map("q$_",(4..7,16..19));
 452 my $j=0;
 453
 454 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
 455 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
 456   my $arg = pop;
 457     $arg = "#$arg" if ($arg*1 eq $arg);
 458     $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
 459 }
 460
 461 sub Dscalar { shift =~ m|[qv]([0-9]+)|?"d$1":""; }
 462 sub Dlo     { shift =~ m|[qv]([0-9]+)|?"v$1.d[0]":""; }
 463 sub Dhi     { shift =~ m|[qv]([0-9]+)|?"v$1.d[1]":""; }
 464
 465 sub Xupdate()
 466 { use integer;
 467   my $body = shift;
 468   my @insns = (&$body,&$body,&$body,&$body);
 469   my ($a,$b,$c,$d,$e,$f,$g,$h);
 470
 471         &ext_8          ($T0,@X[0],@X[1],4);    # X[1..4]
 472          eval(shift(@insns));
 473          eval(shift(@insns));
 474          eval(shift(@insns));
 475         &ext_8          ($T3,@X[2],@X[3],4);    # X[9..12]
 476          eval(shift(@insns));
 477          eval(shift(@insns));
 478         &mov            (&Dscalar($T7),&Dhi(@X[3]));    # X[14..15]
 479          eval(shift(@insns));
 480          eval(shift(@insns));
 481         &ushr_32        ($T2,$T0,$sigma0[0]);
 482          eval(shift(@insns));
 483         &ushr_32        ($T1,$T0,$sigma0[2]);
 484          eval(shift(@insns));
 485         &add_32         (@X[0],@X[0],$T3);      # X[0..3] += X[9..12]
 486          eval(shift(@insns));
 487         &sli_32         ($T2,$T0,32-$sigma0[0]);
 488          eval(shift(@insns));
 489          eval(shift(@insns));
 490         &ushr_32        ($T3,$T0,$sigma0[1]);
 491          eval(shift(@insns));
 492          eval(shift(@insns));
 493         &eor_8          ($T1,$T1,$T2);
 494          eval(shift(@insns));
 495          eval(shift(@insns));
 496         &sli_32         ($T3,$T0,32-$sigma0[1]);
 497          eval(shift(@insns));
 498          eval(shift(@insns));
 499           &ushr_32      ($T4,$T7,$sigma1[0]);
 500          eval(shift(@insns));
 501          eval(shift(@insns));
 502         &eor_8          ($T1,$T1,$T3);          # sigma0(X[1..4])
 503          eval(shift(@insns));
 504          eval(shift(@insns));
 505           &sli_32       ($T4,$T7,32-$sigma1[0]);
 506          eval(shift(@insns));
 507          eval(shift(@insns));
 508           &ushr_32      ($T5,$T7,$sigma1[2]);
 509          eval(shift(@insns));
 510          eval(shift(@insns));
 511           &ushr_32      ($T3,$T7,$sigma1[1]);
 512          eval(shift(@insns));
 513          eval(shift(@insns));
 514         &add_32         (@X[0],@X[0],$T1);      # X[0..3] += sigma0(X[1..4])
 515          eval(shift(@insns));
 516          eval(shift(@insns));
 517           &sli_u32      ($T3,$T7,32-$sigma1[1]);
 518          eval(shift(@insns));
 519          eval(shift(@insns));
 520           &eor_8        ($T5,$T5,$T4);
 521          eval(shift(@insns));
 522          eval(shift(@insns));
 523          eval(shift(@insns));
 524           &eor_8        ($T5,$T5,$T3);          # sigma1(X[14..15])
 525          eval(shift(@insns));
 526          eval(shift(@insns));
 527          eval(shift(@insns));
 528         &add_32         (@X[0],@X[0],$T5);      # X[0..1] += sigma1(X[14..15])
 529          eval(shift(@insns));
 530          eval(shift(@insns));
 531          eval(shift(@insns));
 532           &ushr_32      ($T6,@X[0],$sigma1[0]);
 533          eval(shift(@insns));
 534           &ushr_32      ($T7,@X[0],$sigma1[2]);
 535          eval(shift(@insns));
 536          eval(shift(@insns));
 537           &sli_32       ($T6,@X[0],32-$sigma1[0]);
 538          eval(shift(@insns));
 539           &ushr_32      ($T5,@X[0],$sigma1[1]);
 540          eval(shift(@insns));
 541          eval(shift(@insns));
 542           &eor_8        ($T7,$T7,$T6);
 543          eval(shift(@insns));
 544          eval(shift(@insns));
 545           &sli_32       ($T5,@X[0],32-$sigma1[1]);
 546          eval(shift(@insns));
 547          eval(shift(@insns));
 548         &ld1_32         ("{$T0}","[$Ktbl], #16");
 549          eval(shift(@insns));
 550           &eor_8        ($T7,$T7,$T5);          # sigma1(X[16..17])
 551          eval(shift(@insns));
 552          eval(shift(@insns));
 553         &eor_8          ($T5,$T5,$T5);
 554          eval(shift(@insns));
 555          eval(shift(@insns));
 556         &mov            (&Dhi($T5), &Dlo($T7));
 557          eval(shift(@insns));
 558          eval(shift(@insns));
 559          eval(shift(@insns));
 560         &add_32         (@X[0],@X[0],$T5);      # X[2..3] += sigma1(X[16..17])
 561          eval(shift(@insns));
 562          eval(shift(@insns));
 563          eval(shift(@insns));
 564         &add_32         ($T0,$T0,@X[0]);
 565          while($#insns>=1) { eval(shift(@insns)); }
 566         &st1_32         ("{$T0}","[$Xfer], #16");
 567          eval(shift(@insns));
 568
 569         push(@X,shift(@X));             # "rotate" X[]
 570 }
 571
 572 sub Xpreload()
 573 { use integer;
 574   my $body = shift;
 575   my @insns = (&$body,&$body,&$body,&$body);
 576   my ($a,$b,$c,$d,$e,$f,$g,$h);
 577
 578          eval(shift(@insns));
 579          eval(shift(@insns));
 580         &ld1_8          ("{@X[0]}","[$inp],#16");
 581          eval(shift(@insns));
 582          eval(shift(@insns));
 583         &ld1_32         ("{$T0}","[$Ktbl],#16");
 584          eval(shift(@insns));
 585          eval(shift(@insns));
 586          eval(shift(@insns));
 587          eval(shift(@insns));
 588         &rev32          (@X[0],@X[0]);
 589          eval(shift(@insns));
 590          eval(shift(@insns));
 591          eval(shift(@insns));
 592          eval(shift(@insns));
 593         &add_32         ($T0,$T0,@X[0]);
 594          foreach (@insns) { eval; }     # remaining instructions
 595         &st1_32         ("{$T0}","[$Xfer], #16");
 596
 597         push(@X,shift(@X));             # "rotate" X[]
 598 }
 599
 600 sub body_00_15 () {
 601         (
 602         '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
 603         '&add   ($h,$h,$t1)',                   # h+=X[i]+K[i]
 604         '&add   ($a,$a,$t4);'.                  # h+=Sigma0(a) from the past
 605         '&and   ($t1,$f,$e)',
 606         '&bic   ($t4,$g,$e)',
 607         '&eor   ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
 608         '&add   ($a,$a,$t2)',                   # h+=Maj(a,b,c) from the past
 609         '&orr   ($t1,$t1,$t4)',                 # Ch(e,f,g)
 610         '&eor   ($t0,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))',   # Sigma1(e)
 611         '&eor   ($t4,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
 612         '&add   ($h,$h,$t1)',                   # h+=Ch(e,f,g)
 613         '&ror   ($t0,$t0,"#$Sigma1[0]")',
 614         '&eor   ($t2,$a,$b)',                   # a^b, b^c in next round
 615         '&eor   ($t4,$t4,$a,"ror#".($Sigma0[2]-$Sigma0[0]))',   # Sigma0(a)
 616         '&add   ($h,$h,$t0)',                   # h+=Sigma1(e)
 617         '&ldr   ($t1,sprintf "[sp,#%d]",4*(($j+1)&15))  if (($j&15)!=15);'.
 618         '&ldr   ($t1,"[$Ktbl]")                         if ($j==15);'.
 619         '&and   ($t3,$t3,$t2)',                 # (b^c)&=(a^b)
 620         '&ror   ($t4,$t4,"#$Sigma0[0]")',
 621         '&add   ($d,$d,$h)',                    # d+=h
 622         '&eor   ($t3,$t3,$b)',                  # Maj(a,b,c)
 623         '$j++;  unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
 624         )
 625 }
 626
 627 $code.=<<___;
 628 #ifdef  __KERNEL__
 629 .globl  sha256_block_neon
 630 #endif
 631 .type   sha256_block_neon,%function
 632 .align  4
 633 sha256_block_neon:
 634 .Lneon_entry:
 635         stp     x29, x30, [sp, #-16]!
 636         mov     x29, sp
 637         sub     sp,sp,#16*4
 638
 639         adr     $Ktbl,.LK256
 640         add     $num,$inp,$num,lsl#6    // len to point at the end of inp
 641
 642         ld1.8   {@X[0]},[$inp], #16
 643         ld1.8   {@X[1]},[$inp], #16
 644         ld1.8   {@X[2]},[$inp], #16
 645         ld1.8   {@X[3]},[$inp], #16
 646         ld1.32  {$T0},[$Ktbl], #16
 647         ld1.32  {$T1},[$Ktbl], #16
 648         ld1.32  {$T2},[$Ktbl], #16
 649         ld1.32  {$T3},[$Ktbl], #16
 650         rev32   @X[0],@X[0]             // yes, even on
 651         rev32   @X[1],@X[1]             // big-endian
 652         rev32   @X[2],@X[2]
 653         rev32   @X[3],@X[3]
 654         mov     $Xfer,sp
 655         add.32  $T0,$T0,@X[0]
 656         add.32  $T1,$T1,@X[1]
 657         add.32  $T2,$T2,@X[2]
 658         st1.32  {$T0-$T1},[$Xfer], #32
 659         add.32  $T3,$T3,@X[3]
 660         st1.32  {$T2-$T3},[$Xfer]
 661         sub     $Xfer,$Xfer,#32
 662
 663         ldp     $A,$B,[$ctx]
 664         ldp     $C,$D,[$ctx,#8]
 665         ldp     $E,$F,[$ctx,#16]
 666         ldp     $G,$H,[$ctx,#24]
 667         ldr     $t1,[sp,#0]
 668         mov     $t2,wzr
 669         eor     $t3,$B,$C
 670         mov     $t4,wzr
 671         b       .L_00_48
 672
 673 .align  4
 674 .L_00_48:
 675 ___
 676         &Xupdate(\&body_00_15);
 677         &Xupdate(\&body_00_15);
 678         &Xupdate(\&body_00_15);
 679         &Xupdate(\&body_00_15);
 680 $code.=<<___;
 681         cmp     $t1,#0                          // check for K256 terminator
 682         ldr     $t1,[sp,#0]
 683         sub     $Xfer,$Xfer,#64
 684         bne     .L_00_48
 685
 686         sub     $Ktbl,$Ktbl,#256                // rewind $Ktbl
 687         cmp     $inp,$num
 688         mov     $Xfer, #64
 689         csel    $Xfer, $Xfer, xzr, eq
 690         sub     $inp,$inp,$Xfer                 // avoid SEGV
 691         mov     $Xfer,sp
 692 ___
 693         &Xpreload(\&body_00_15);
 694         &Xpreload(\&body_00_15);
 695         &Xpreload(\&body_00_15);
 696         &Xpreload(\&body_00_15);
 697 $code.=<<___;
 698         add     $A,$A,$t4                       // h+=Sigma0(a) from the past
 699         ldp     $t0,$t1,[$ctx,#0]
 700         add     $A,$A,$t2                       // h+=Maj(a,b,c) from the past
 701         ldp     $t2,$t3,[$ctx,#8]
 702         add     $A,$A,$t0                       // accumulate
 703         add     $B,$B,$t1
 704         ldp     $t0,$t1,[$ctx,#16]
 705         add     $C,$C,$t2
 706         add     $D,$D,$t3
 707         ldp     $t2,$t3,[$ctx,#24]
 708         add     $E,$E,$t0
 709         add     $F,$F,$t1
 710          ldr    $t1,[sp,#0]
 711         stp     $A,$B,[$ctx,#0]
 712         add     $G,$G,$t2
 713          mov    $t2,wzr
 714         stp     $C,$D,[$ctx,#8]
 715         add     $H,$H,$t3
 716         stp     $E,$F,[$ctx,#16]
 717          eor    $t3,$B,$C
 718         stp     $G,$H,[$ctx,#24]
 719          mov    $t4,wzr
 720          mov    $Xfer,sp
 721         b.ne    .L_00_48
 722
 723         ldr     x29,[x29]
 724         add     sp,sp,#16*4+16
 725         ret
 726 .size   sha256_block_neon,.-sha256_block_neon
 727 ___
 728 }
 729
 730 if ($SZ==8) {
 731 my $Ktbl="x3";
 732
 733 my @H = map("v$_.16b",(0..4));
 734 my ($fg,$de,$m9_10)=map("v$_.16b",(5..7));
 735 my @MSG=map("v$_.16b",(16..23));
 736 my ($W0,$W1)=("v24.2d","v25.2d");
 737 my ($AB,$CD,$EF,$GH)=map("v$_.16b",(26..29));
 738
 739 $code.=<<___;
 740 #ifndef __KERNEL__
 741 .type   sha512_block_armv8,%function
 742 .align  6
 743 sha512_block_armv8:
 744 .Lv8_entry:
 745         stp             x29,x30,[sp,#-16]!
 746         add             x29,sp,#0
 747
 748         ld1             {@MSG[0]-@MSG[3]},[$inp],#64    // load input
 749         ld1             {@MSG[4]-@MSG[7]},[$inp],#64
 750
 751         ld1.64          {@H[0]-@H[3]},[$ctx]            // load context
 752         adr             $Ktbl,.LK512
 753
 754         rev64           @MSG[0],@MSG[0]
 755         rev64           @MSG[1],@MSG[1]
 756         rev64           @MSG[2],@MSG[2]
 757         rev64           @MSG[3],@MSG[3]
 758         rev64           @MSG[4],@MSG[4]
 759         rev64           @MSG[5],@MSG[5]
 760         rev64           @MSG[6],@MSG[6]
 761         rev64           @MSG[7],@MSG[7]
 762         b               .Loop_hw
 763
 764 .align  4
 765 .Loop_hw:
 766         ld1.64          {$W0},[$Ktbl],#16
 767         subs            $num,$num,#1
 768         sub             x4,$inp,#128
 769         orr             $AB,@H[0],@H[0]                 // offload
 770         orr             $CD,@H[1],@H[1]
 771         orr             $EF,@H[2],@H[2]
 772         orr             $GH,@H[3],@H[3]
 773         csel            $inp,$inp,x4,ne                 // conditional rewind
 774 ___
 775 for($i=0;$i<32;$i++) {
 776 $code.=<<___;
 777         add.i64         $W0,$W0,@MSG[0]
 778         ld1.64          {$W1},[$Ktbl],#16
 779         ext             $W0,$W0,$W0,#8
 780         ext             $fg,@H[2],@H[3],#8
 781         ext             $de,@H[1],@H[2],#8
 782         add.i64         @H[3],@H[3],$W0                 // "T1 + H + K512[i]"
 783          sha512su0      @MSG[0],@MSG[1]
 784          ext            $m9_10,@MSG[4],@MSG[5],#8
 785         sha512h         @H[3],$fg,$de
 786          sha512su1      @MSG[0],@MSG[7],$m9_10
 787         add.i64         @H[4],@H[1],@H[3]               // "D + T1"
 788         sha512h2        @H[3],$H[1],@H[0]
 789 ___
 790         ($W0,$W1)=($W1,$W0);    push(@MSG,shift(@MSG));
 791         @H = (@H[3],@H[0],@H[4],@H[2],@H[1]);
 792 }
 793 for(;$i<40;$i++) {
 794 $code.=<<___    if ($i<39);
 795         ld1.64          {$W1},[$Ktbl],#16
 796 ___
 797 $code.=<<___    if ($i==39);
 798         sub             $Ktbl,$Ktbl,#$rounds*$SZ        // rewind
 799 ___
 800 $code.=<<___;
 801         add.i64         $W0,$W0,@MSG[0]
 802          ld1            {@MSG[0]},[$inp],#16            // load next input
 803         ext             $W0,$W0,$W0,#8
 804         ext             $fg,@H[2],@H[3],#8
 805         ext             $de,@H[1],@H[2],#8
 806         add.i64         @H[3],@H[3],$W0                 // "T1 + H + K512[i]"
 807         sha512h         @H[3],$fg,$de
 808          rev64          @MSG[0],@MSG[0]
 809         add.i64         @H[4],@H[1],@H[3]               // "D + T1"
 810         sha512h2        @H[3],$H[1],@H[0]
 811 ___
 812         ($W0,$W1)=($W1,$W0);    push(@MSG,shift(@MSG));
 813         @H = (@H[3],@H[0],@H[4],@H[2],@H[1]);
 814 }
 815 $code.=<<___;
 816         add.i64         @H[0],@H[0],$AB                 // accumulate
 817         add.i64         @H[1],@H[1],$CD
 818         add.i64         @H[2],@H[2],$EF
 819         add.i64         @H[3],@H[3],$GH
 820
 821         cbnz            $num,.Loop_hw
 822
 823         st1.64          {@H[0]-@H[3]},[$ctx]            // store context
 824
 825         ldr             x29,[sp],#16
 826         ret
 827 .size   sha512_block_armv8,.-sha512_block_armv8
 828 #endif
 829 ___
 830 }
 831
 832 $code.=<<___;
 833 #if !defined(__KERNEL__) && !defined(_WIN64)
 834 .comm   OPENSSL_armcap_P,4,4
 835 #endif
 836 ___
 837
 838 {   my  %opcode = (
 839         "sha256h"       => 0x5e004000,  "sha256h2"      => 0x5e005000,
 840         "sha256su0"     => 0x5e282800,  "sha256su1"     => 0x5e006000   );
 841
 842     sub unsha256 {
 843         my ($mnemonic,$arg)=@_;
 844
 845         $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
 846         &&
 847         sprintf ".inst\t0x%08x\t//%s %s",
 848                         $opcode{$mnemonic}|$1|($2<<5)|($3<<16),
 849                         $mnemonic,$arg;
 850     }
 851 }
 852
 853 {   my  %opcode = (
 854         "sha512h"       => 0xce608000,  "sha512h2"      => 0xce608400,
 855         "sha512su0"     => 0xcec08000,  "sha512su1"     => 0xce608800   );
 856
 857     sub unsha512 {
 858         my ($mnemonic,$arg)=@_;
 859
 860         $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
 861         &&
 862         sprintf ".inst\t0x%08x\t//%s %s",
 863                         $opcode{$mnemonic}|$1|($2<<5)|($3<<16),
 864                         $mnemonic,$arg;
 865     }
 866 }
 867
 868 open SELF,$0;
 869 while(<SELF>) {
 870         next if (/^#!/);
 871         last if (!s/^#/\/\// and !/^$/);
 872         print;
 873 }
 874 close SELF;
 875
 876 foreach(split("\n",$code)) {
 877
 878         s/\`([^\`]*)\`/eval($1)/ge;
 879
 880         s/\b(sha512\w+)\s+([qv].*)/unsha512($1,$2)/ge   or
 881         s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/ge;
 882
 883         s/\bq([0-9]+)\b/v$1.16b/g;              # old->new registers
 884
 885         s/\.[ui]?8(\s)/$1/;
 886         s/\.\w?64\b//           and s/\.16b/\.2d/g      or
 887         s/\.\w?32\b//           and s/\.16b/\.4s/g;
 888         m/\bext\b/              and s/\.2d/\.16b/g      or
 889         m/(ld|st)1[^\[]+\[0\]/  and s/\.4s/\.s/g;
 890
 891         print $_,"\n";
 892 }
 893
 894 close STDOUT;