crypto/chacha/asm/chacha-armv8.pl

   1 #!/usr/bin/env perl
   2 #
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9 #
  10 # June 2015
  11 #
  12 # ChaCha20 for ARMv8.
  13 #
  14 # Performance in cycles per byte out of large buffer.
  15 #
  16 #                       IALU/gcc-4.9    3xNEON+1xIALU   6xNEON+2xIALU
  17 #
  18 # Apple A7              5.50/+49%       3.33            1.70
  19 # Cortex-A53            8.40/+80%       4.72            4.72(*)
  20 # Cortex-A57            8.06/+43%       4.90            4.43(**)
  21 # Denver                4.50/+82%       2.63            2.67(*)
  22 # X-Gene                9.50/+46%       8.82            8.89(*)
  23 #
  24 # (*)   it's expected that doubling interleave factor doesn't help
  25 #       all processors, only those with higher NEON latency and
  26 #       higher instruction issue rate;
  27 # (**)  expected improvement was actually higher;
  28
  29 $flavour=shift;
  30 $output=shift;
  31
  32 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  33 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  34 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  35 die "can't locate arm-xlate.pl";
  36
  37 open OUT,"| \"$^X\" $xlate $flavour $output";
  38 *STDOUT=*OUT;
  39
  40 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
  41 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
  42   my $arg = pop;
  43     $arg = "#$arg" if ($arg*1 eq $arg);
  44     $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
  45 }
  46
  47 my ($out,$inp,$len,$key,$ctr) = map("x$_",(0..4));
  48
  49 my @x=map("x$_",(5..17,19..21));
  50 my @d=map("x$_",(22..28,30));
  51
  52 sub ROUND {
  53 my ($a0,$b0,$c0,$d0)=@_;
  54 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
  55 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
  56 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
  57
  58     (
  59         "&add_32        (@x[$a0],@x[$a0],@x[$b0])",
  60          "&add_32       (@x[$a1],@x[$a1],@x[$b1])",
  61           "&add_32      (@x[$a2],@x[$a2],@x[$b2])",
  62            "&add_32     (@x[$a3],@x[$a3],@x[$b3])",
  63         "&eor_32        (@x[$d0],@x[$d0],@x[$a0])",
  64          "&eor_32       (@x[$d1],@x[$d1],@x[$a1])",
  65           "&eor_32      (@x[$d2],@x[$d2],@x[$a2])",
  66            "&eor_32     (@x[$d3],@x[$d3],@x[$a3])",
  67         "&ror_32        (@x[$d0],@x[$d0],16)",
  68          "&ror_32       (@x[$d1],@x[$d1],16)",
  69           "&ror_32      (@x[$d2],@x[$d2],16)",
  70            "&ror_32     (@x[$d3],@x[$d3],16)",
  71
  72         "&add_32        (@x[$c0],@x[$c0],@x[$d0])",
  73          "&add_32       (@x[$c1],@x[$c1],@x[$d1])",
  74           "&add_32      (@x[$c2],@x[$c2],@x[$d2])",
  75            "&add_32     (@x[$c3],@x[$c3],@x[$d3])",
  76         "&eor_32        (@x[$b0],@x[$b0],@x[$c0])",
  77          "&eor_32       (@x[$b1],@x[$b1],@x[$c1])",
  78           "&eor_32      (@x[$b2],@x[$b2],@x[$c2])",
  79            "&eor_32     (@x[$b3],@x[$b3],@x[$c3])",
  80         "&ror_32        (@x[$b0],@x[$b0],20)",
  81          "&ror_32       (@x[$b1],@x[$b1],20)",
  82           "&ror_32      (@x[$b2],@x[$b2],20)",
  83            "&ror_32     (@x[$b3],@x[$b3],20)",
  84
  85         "&add_32        (@x[$a0],@x[$a0],@x[$b0])",
  86          "&add_32       (@x[$a1],@x[$a1],@x[$b1])",
  87           "&add_32      (@x[$a2],@x[$a2],@x[$b2])",
  88            "&add_32     (@x[$a3],@x[$a3],@x[$b3])",
  89         "&eor_32        (@x[$d0],@x[$d0],@x[$a0])",
  90          "&eor_32       (@x[$d1],@x[$d1],@x[$a1])",
  91           "&eor_32      (@x[$d2],@x[$d2],@x[$a2])",
  92            "&eor_32     (@x[$d3],@x[$d3],@x[$a3])",
  93         "&ror_32        (@x[$d0],@x[$d0],24)",
  94          "&ror_32       (@x[$d1],@x[$d1],24)",
  95           "&ror_32      (@x[$d2],@x[$d2],24)",
  96            "&ror_32     (@x[$d3],@x[$d3],24)",
  97
  98         "&add_32        (@x[$c0],@x[$c0],@x[$d0])",
  99          "&add_32       (@x[$c1],@x[$c1],@x[$d1])",
 100           "&add_32      (@x[$c2],@x[$c2],@x[$d2])",
 101            "&add_32     (@x[$c3],@x[$c3],@x[$d3])",
 102         "&eor_32        (@x[$b0],@x[$b0],@x[$c0])",
 103          "&eor_32       (@x[$b1],@x[$b1],@x[$c1])",
 104           "&eor_32      (@x[$b2],@x[$b2],@x[$c2])",
 105            "&eor_32     (@x[$b3],@x[$b3],@x[$c3])",
 106         "&ror_32        (@x[$b0],@x[$b0],25)",
 107          "&ror_32       (@x[$b1],@x[$b1],25)",
 108           "&ror_32      (@x[$b2],@x[$b2],25)",
 109            "&ror_32     (@x[$b3],@x[$b3],25)"
 110     );
 111 }
 112
 113 $code.=<<___;
 114 #include "arm_arch.h"
 115
 116 .text
 117
 118 .extern OPENSSL_armcap_P
 119
 120 .align  5
 121 .Lsigma:
 122 .quad   0x3320646e61707865,0x6b20657479622d32           // endian-neutral
 123 .Lone:
 124 .long   1,0,0,0
 125 .LOPENSSL_armcap_P:
 126 #ifdef  __ILP32__
 127 .long   OPENSSL_armcap_P-.
 128 #else
 129 .quad   OPENSSL_armcap_P-.
 130 #endif
 131 .asciz  "ChaCha20 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
 132
 133 .globl  ChaCha20_ctr32
 134 .type   ChaCha20_ctr32,%function
 135 .align  5
 136 ChaCha20_ctr32:
 137         cbz     $len,.Labort
 138         adr     @x[0],.LOPENSSL_armcap_P
 139         cmp     $len,#192
 140         b.lo    .Lshort
 141 #ifdef  __ILP32__
 142         ldrsw   @x[1],[@x[0]]
 143 #else
 144         ldr     @x[1],[@x[0]]
 145 #endif
 146         ldr     w17,[@x[1],@x[0]]
 147         tst     w17,#ARMV7_NEON
 148         b.ne    ChaCha20_neon
 149
 150 .Lshort:
 151         stp     x29,x30,[sp,#-96]!
 152         add     x29,sp,#0
 153
 154         adr     @x[0],.Lsigma
 155         stp     x19,x20,[sp,#16]
 156         stp     x21,x22,[sp,#32]
 157         stp     x23,x24,[sp,#48]
 158         stp     x25,x26,[sp,#64]
 159         stp     x27,x28,[sp,#80]
 160         sub     sp,sp,#64
 161
 162         ldp     @d[0],@d[1],[@x[0]]             // load sigma
 163         ldp     @d[2],@d[3],[$key]              // load key
 164         ldp     @d[4],@d[5],[$key,#16]
 165         ldp     @d[6],@d[7],[$ctr]              // load counter
 166 #ifdef  __ARMEB__
 167         ror     @d[2],@d[2],#32
 168         ror     @d[3],@d[3],#32
 169         ror     @d[4],@d[4],#32
 170         ror     @d[5],@d[5],#32
 171         ror     @d[6],@d[6],#32
 172         ror     @d[7],@d[7],#32
 173 #endif
 174
 175 .Loop_outer:
 176         mov.32  @x[0],@d[0]                     // unpack key block
 177         lsr     @x[1],@d[0],#32
 178         mov.32  @x[2],@d[1]
 179         lsr     @x[3],@d[1],#32
 180         mov.32  @x[4],@d[2]
 181         lsr     @x[5],@d[2],#32
 182         mov.32  @x[6],@d[3]
 183         lsr     @x[7],@d[3],#32
 184         mov.32  @x[8],@d[4]
 185         lsr     @x[9],@d[4],#32
 186         mov.32  @x[10],@d[5]
 187         lsr     @x[11],@d[5],#32
 188         mov.32  @x[12],@d[6]
 189         lsr     @x[13],@d[6],#32
 190         mov.32  @x[14],@d[7]
 191         lsr     @x[15],@d[7],#32
 192
 193         mov     $ctr,#10
 194         subs    $len,$len,#64
 195 .Loop:
 196         sub     $ctr,$ctr,#1
 197 ___
 198         foreach (&ROUND(0, 4, 8,12)) { eval; }
 199         foreach (&ROUND(0, 5,10,15)) { eval; }
 200 $code.=<<___;
 201         cbnz    $ctr,.Loop
 202
 203         add.32  @x[0],@x[0],@d[0]               // accumulate key block
 204         add     @x[1],@x[1],@d[0],lsr#32
 205         add.32  @x[2],@x[2],@d[1]
 206         add     @x[3],@x[3],@d[1],lsr#32
 207         add.32  @x[4],@x[4],@d[2]
 208         add     @x[5],@x[5],@d[2],lsr#32
 209         add.32  @x[6],@x[6],@d[3]
 210         add     @x[7],@x[7],@d[3],lsr#32
 211         add.32  @x[8],@x[8],@d[4]
 212         add     @x[9],@x[9],@d[4],lsr#32
 213         add.32  @x[10],@x[10],@d[5]
 214         add     @x[11],@x[11],@d[5],lsr#32
 215         add.32  @x[12],@x[12],@d[6]
 216         add     @x[13],@x[13],@d[6],lsr#32
 217         add.32  @x[14],@x[14],@d[7]
 218         add     @x[15],@x[15],@d[7],lsr#32
 219
 220         b.lo    .Ltail
 221
 222         add     @x[0],@x[0],@x[1],lsl#32        // pack
 223         add     @x[2],@x[2],@x[3],lsl#32
 224         ldp     @x[1],@x[3],[$inp,#0]           // load input
 225         add     @x[4],@x[4],@x[5],lsl#32
 226         add     @x[6],@x[6],@x[7],lsl#32
 227         ldp     @x[5],@x[7],[$inp,#16]
 228         add     @x[8],@x[8],@x[9],lsl#32
 229         add     @x[10],@x[10],@x[11],lsl#32
 230         ldp     @x[9],@x[11],[$inp,#32]
 231         add     @x[12],@x[12],@x[13],lsl#32
 232         add     @x[14],@x[14],@x[15],lsl#32
 233         ldp     @x[13],@x[15],[$inp,#48]
 234         add     $inp,$inp,#64
 235 #ifdef  __ARMEB__
 236         rev     @x[0],@x[0]
 237         rev     @x[2],@x[2]
 238         rev     @x[4],@x[4]
 239         rev     @x[6],@x[6]
 240         rev     @x[8],@x[8]
 241         rev     @x[10],@x[10]
 242         rev     @x[12],@x[12]
 243         rev     @x[14],@x[14]
 244 #endif
 245         eor     @x[0],@x[0],@x[1]
 246         eor     @x[2],@x[2],@x[3]
 247         eor     @x[4],@x[4],@x[5]
 248         eor     @x[6],@x[6],@x[7]
 249         eor     @x[8],@x[8],@x[9]
 250         eor     @x[10],@x[10],@x[11]
 251         eor     @x[12],@x[12],@x[13]
 252         eor     @x[14],@x[14],@x[15]
 253
 254         stp     @x[0],@x[2],[$out,#0]           // store output
 255          add    @d[6],@d[6],#1                  // increment counter
 256         stp     @x[4],@x[6],[$out,#16]
 257         stp     @x[8],@x[10],[$out,#32]
 258         stp     @x[12],@x[14],[$out,#48]
 259         add     $out,$out,#64
 260
 261         b.hi    .Loop_outer
 262
 263         ldp     x19,x20,[x29,#16]
 264         add     sp,sp,#64
 265         ldp     x21,x22,[x29,#32]
 266         ldp     x23,x24,[x29,#48]
 267         ldp     x25,x26,[x29,#64]
 268         ldp     x27,x28,[x29,#80]
 269         ldp     x29,x30,[sp],#96
 270 .Labort:
 271         ret
 272
 273 .align  4
 274 .Ltail:
 275         add     $len,$len,#64
 276 .Less_than_64:
 277         sub     $out,$out,#1
 278         add     $inp,$inp,$len
 279         add     $out,$out,$len
 280         add     $ctr,sp,$len
 281         neg     $len,$len
 282
 283         add     @x[0],@x[0],@x[1],lsl#32        // pack
 284         add     @x[2],@x[2],@x[3],lsl#32
 285         add     @x[4],@x[4],@x[5],lsl#32
 286         add     @x[6],@x[6],@x[7],lsl#32
 287         add     @x[8],@x[8],@x[9],lsl#32
 288         add     @x[10],@x[10],@x[11],lsl#32
 289         add     @x[12],@x[12],@x[13],lsl#32
 290         add     @x[14],@x[14],@x[15],lsl#32
 291 #ifdef  __ARMEB__
 292         rev     @x[0],@x[0]
 293         rev     @x[2],@x[2]
 294         rev     @x[4],@x[4]
 295         rev     @x[6],@x[6]
 296         rev     @x[8],@x[8]
 297         rev     @x[10],@x[10]
 298         rev     @x[12],@x[12]
 299         rev     @x[14],@x[14]
 300 #endif
 301         stp     @x[0],@x[2],[sp,#0]
 302         stp     @x[4],@x[6],[sp,#16]
 303         stp     @x[8],@x[10],[sp,#32]
 304         stp     @x[12],@x[14],[sp,#48]
 305
 306 .Loop_tail:
 307         ldrb    w10,[$inp,$len]
 308         ldrb    w11,[$ctr,$len]
 309         add     $len,$len,#1
 310         eor     w10,w10,w11
 311         strb    w10,[$out,$len]
 312         cbnz    $len,.Loop_tail
 313
 314         stp     xzr,xzr,[sp,#0]
 315         stp     xzr,xzr,[sp,#16]
 316         stp     xzr,xzr,[sp,#32]
 317         stp     xzr,xzr,[sp,#48]
 318
 319         ldp     x19,x20,[x29,#16]
 320         add     sp,sp,#64
 321         ldp     x21,x22,[x29,#32]
 322         ldp     x23,x24,[x29,#48]
 323         ldp     x25,x26,[x29,#64]
 324         ldp     x27,x28,[x29,#80]
 325         ldp     x29,x30,[sp],#96
 326         ret
 327 .size   ChaCha20_ctr32,.-ChaCha20_ctr32
 328 ___
 329
 330 {{{
 331 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2,$T3) =
 332     map("v$_.4s",(0..7,16..23));
 333 my (@K)=map("v$_.4s",(24..30));
 334 my $ONE="v31.4s";
 335
 336 sub NEONROUND {
 337 my $odd = pop;
 338 my ($a,$b,$c,$d,$t)=@_;
 339
 340         (
 341         "&add           ('$a','$a','$b')",
 342         "&eor           ('$d','$d','$a')",
 343         "&rev32_16      ('$d','$d')",           # vrot ($d,16)
 344
 345         "&add           ('$c','$c','$d')",
 346         "&eor           ('$t','$b','$c')",
 347         "&ushr          ('$b','$t',20)",
 348         "&sli           ('$b','$t',12)",
 349
 350         "&add           ('$a','$a','$b')",
 351         "&eor           ('$t','$d','$a')",
 352         "&ushr          ('$d','$t',24)",
 353         "&sli           ('$d','$t',8)",
 354
 355         "&add           ('$c','$c','$d')",
 356         "&eor           ('$t','$b','$c')",
 357         "&ushr          ('$b','$t',25)",
 358         "&sli           ('$b','$t',7)",
 359
 360         "&ext           ('$c','$c','$c',8)",
 361         "&ext           ('$d','$d','$d',$odd?4:12)",
 362         "&ext           ('$b','$b','$b',$odd?12:4)"
 363         );
 364 }
 365
 366 $code.=<<___;
 367
 368 .type   ChaCha20_neon,%function
 369 .align  5
 370 ChaCha20_neon:
 371         stp     x29,x30,[sp,#-96]!
 372         add     x29,sp,#0
 373
 374         adr     @x[0],.Lsigma
 375         stp     x19,x20,[sp,#16]
 376         stp     x21,x22,[sp,#32]
 377         stp     x23,x24,[sp,#48]
 378         stp     x25,x26,[sp,#64]
 379         stp     x27,x28,[sp,#80]
 380         cmp     $len,#512
 381         b.hs    .L512_or_more_neon
 382
 383         sub     sp,sp,#64
 384
 385         ldp     @d[0],@d[1],[@x[0]]             // load sigma
 386         ld1     {@K[0]},[@x[0]],#16
 387         ldp     @d[2],@d[3],[$key]              // load key
 388         ldp     @d[4],@d[5],[$key,#16]
 389         ld1     {@K[1],@K[2]},[$key]
 390         ldp     @d[6],@d[7],[$ctr]              // load counter
 391         ld1     {@K[3]},[$ctr]
 392         ld1     {$ONE},[@x[0]]
 393 #ifdef  __ARMEB__
 394         rev64   @K[0],@K[0]
 395         ror     @d[2],@d[2],#32
 396         ror     @d[3],@d[3],#32
 397         ror     @d[4],@d[4],#32
 398         ror     @d[5],@d[5],#32
 399         ror     @d[6],@d[6],#32
 400         ror     @d[7],@d[7],#32
 401 #endif
 402         add     @K[3],@K[3],$ONE                // += 1
 403         add     @K[4],@K[3],$ONE
 404         add     @K[5],@K[4],$ONE
 405         shl     $ONE,$ONE,#2                    // 1 -> 4
 406
 407 .Loop_outer_neon:
 408         mov.32  @x[0],@d[0]                     // unpack key block
 409         lsr     @x[1],@d[0],#32
 410          mov    $A0,@K[0]
 411         mov.32  @x[2],@d[1]
 412         lsr     @x[3],@d[1],#32
 413          mov    $A1,@K[0]
 414         mov.32  @x[4],@d[2]
 415         lsr     @x[5],@d[2],#32
 416          mov    $A2,@K[0]
 417         mov.32  @x[6],@d[3]
 418          mov    $B0,@K[1]
 419         lsr     @x[7],@d[3],#32
 420          mov    $B1,@K[1]
 421         mov.32  @x[8],@d[4]
 422          mov    $B2,@K[1]
 423         lsr     @x[9],@d[4],#32
 424          mov    $D0,@K[3]
 425         mov.32  @x[10],@d[5]
 426          mov    $D1,@K[4]
 427         lsr     @x[11],@d[5],#32
 428          mov    $D2,@K[5]
 429         mov.32  @x[12],@d[6]
 430          mov    $C0,@K[2]
 431         lsr     @x[13],@d[6],#32
 432          mov    $C1,@K[2]
 433         mov.32  @x[14],@d[7]
 434          mov    $C2,@K[2]
 435         lsr     @x[15],@d[7],#32
 436
 437         mov     $ctr,#10
 438         subs    $len,$len,#256
 439 .Loop_neon:
 440         sub     $ctr,$ctr,#1
 441 ___
 442         my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
 443         my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
 444         my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
 445         my @thread3=&ROUND(0,4,8,12);
 446
 447         foreach (@thread0) {
 448                 eval;                   eval(shift(@thread3));
 449                 eval(shift(@thread1));  eval(shift(@thread3));
 450                 eval(shift(@thread2));  eval(shift(@thread3));
 451         }
 452
 453         @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
 454         @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
 455         @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
 456         @thread3=&ROUND(0,5,10,15);
 457
 458         foreach (@thread0) {
 459                 eval;                   eval(shift(@thread3));
 460                 eval(shift(@thread1));  eval(shift(@thread3));
 461                 eval(shift(@thread2));  eval(shift(@thread3));
 462         }
 463 $code.=<<___;
 464         cbnz    $ctr,.Loop_neon
 465
 466         add.32  @x[0],@x[0],@d[0]               // accumulate key block
 467          add    $A0,$A0,@K[0]
 468         add     @x[1],@x[1],@d[0],lsr#32
 469          add    $A1,$A1,@K[0]
 470         add.32  @x[2],@x[2],@d[1]
 471          add    $A2,$A2,@K[0]
 472         add     @x[3],@x[3],@d[1],lsr#32
 473          add    $C0,$C0,@K[2]
 474         add.32  @x[4],@x[4],@d[2]
 475          add    $C1,$C1,@K[2]
 476         add     @x[5],@x[5],@d[2],lsr#32
 477          add    $C2,$C2,@K[2]
 478         add.32  @x[6],@x[6],@d[3]
 479          add    $D0,$D0,@K[3]
 480         add     @x[7],@x[7],@d[3],lsr#32
 481         add.32  @x[8],@x[8],@d[4]
 482          add    $D1,$D1,@K[4]
 483         add     @x[9],@x[9],@d[4],lsr#32
 484         add.32  @x[10],@x[10],@d[5]
 485          add    $D2,$D2,@K[5]
 486         add     @x[11],@x[11],@d[5],lsr#32
 487         add.32  @x[12],@x[12],@d[6]
 488          add    $B0,$B0,@K[1]
 489         add     @x[13],@x[13],@d[6],lsr#32
 490         add.32  @x[14],@x[14],@d[7]
 491          add    $B1,$B1,@K[1]
 492         add     @x[15],@x[15],@d[7],lsr#32
 493          add    $B2,$B2,@K[1]
 494
 495         b.lo    .Ltail_neon
 496
 497         add     @x[0],@x[0],@x[1],lsl#32        // pack
 498         add     @x[2],@x[2],@x[3],lsl#32
 499         ldp     @x[1],@x[3],[$inp,#0]           // load input
 500         add     @x[4],@x[4],@x[5],lsl#32
 501         add     @x[6],@x[6],@x[7],lsl#32
 502         ldp     @x[5],@x[7],[$inp,#16]
 503         add     @x[8],@x[8],@x[9],lsl#32
 504         add     @x[10],@x[10],@x[11],lsl#32
 505         ldp     @x[9],@x[11],[$inp,#32]
 506         add     @x[12],@x[12],@x[13],lsl#32
 507         add     @x[14],@x[14],@x[15],lsl#32
 508         ldp     @x[13],@x[15],[$inp,#48]
 509         add     $inp,$inp,#64
 510 #ifdef  __ARMEB__
 511         rev     @x[0],@x[0]
 512         rev     @x[2],@x[2]
 513         rev     @x[4],@x[4]
 514         rev     @x[6],@x[6]
 515         rev     @x[8],@x[8]
 516         rev     @x[10],@x[10]
 517         rev     @x[12],@x[12]
 518         rev     @x[14],@x[14]
 519 #endif
 520         ld1.8   {$T0-$T3},[$inp],#64
 521         eor     @x[0],@x[0],@x[1]
 522         eor     @x[2],@x[2],@x[3]
 523         eor     @x[4],@x[4],@x[5]
 524         eor     @x[6],@x[6],@x[7]
 525         eor     @x[8],@x[8],@x[9]
 526          eor    $A0,$A0,$T0
 527         eor     @x[10],@x[10],@x[11]
 528          eor    $B0,$B0,$T1
 529         eor     @x[12],@x[12],@x[13]
 530          eor    $C0,$C0,$T2
 531         eor     @x[14],@x[14],@x[15]
 532          eor    $D0,$D0,$T3
 533          ld1.8  {$T0-$T3},[$inp],#64
 534
 535         stp     @x[0],@x[2],[$out,#0]           // store output
 536          add    @d[6],@d[6],#4                  // increment counter
 537         stp     @x[4],@x[6],[$out,#16]
 538          add    @K[3],@K[3],$ONE                // += 4
 539         stp     @x[8],@x[10],[$out,#32]
 540          add    @K[4],@K[4],$ONE
 541         stp     @x[12],@x[14],[$out,#48]
 542          add    @K[5],@K[5],$ONE
 543         add     $out,$out,#64
 544
 545         st1.8   {$A0-$D0},[$out],#64
 546         ld1.8   {$A0-$D0},[$inp],#64
 547
 548         eor     $A1,$A1,$T0
 549         eor     $B1,$B1,$T1
 550         eor     $C1,$C1,$T2
 551         eor     $D1,$D1,$T3
 552         st1.8   {$A1-$D1},[$out],#64
 553
 554         eor     $A2,$A2,$A0
 555         eor     $B2,$B2,$B0
 556         eor     $C2,$C2,$C0
 557         eor     $D2,$D2,$D0
 558         st1.8   {$A2-$D2},[$out],#64
 559
 560         b.hi    .Loop_outer_neon
 561
 562         ldp     x19,x20,[x29,#16]
 563         add     sp,sp,#64
 564         ldp     x21,x22,[x29,#32]
 565         ldp     x23,x24,[x29,#48]
 566         ldp     x25,x26,[x29,#64]
 567         ldp     x27,x28,[x29,#80]
 568         ldp     x29,x30,[sp],#96
 569         ret
 570
 571 .Ltail_neon:
 572         add     $len,$len,#256
 573         cmp     $len,#64
 574         b.lo    .Less_than_64
 575
 576         add     @x[0],@x[0],@x[1],lsl#32        // pack
 577         add     @x[2],@x[2],@x[3],lsl#32
 578         ldp     @x[1],@x[3],[$inp,#0]           // load input
 579         add     @x[4],@x[4],@x[5],lsl#32
 580         add     @x[6],@x[6],@x[7],lsl#32
 581         ldp     @x[5],@x[7],[$inp,#16]
 582         add     @x[8],@x[8],@x[9],lsl#32
 583         add     @x[10],@x[10],@x[11],lsl#32
 584         ldp     @x[9],@x[11],[$inp,#32]
 585         add     @x[12],@x[12],@x[13],lsl#32
 586         add     @x[14],@x[14],@x[15],lsl#32
 587         ldp     @x[13],@x[15],[$inp,#48]
 588         add     $inp,$inp,#64
 589 #ifdef  __ARMEB__
 590         rev     @x[0],@x[0]
 591         rev     @x[2],@x[2]
 592         rev     @x[4],@x[4]
 593         rev     @x[6],@x[6]
 594         rev     @x[8],@x[8]
 595         rev     @x[10],@x[10]
 596         rev     @x[12],@x[12]
 597         rev     @x[14],@x[14]
 598 #endif
 599         eor     @x[0],@x[0],@x[1]
 600         eor     @x[2],@x[2],@x[3]
 601         eor     @x[4],@x[4],@x[5]
 602         eor     @x[6],@x[6],@x[7]
 603         eor     @x[8],@x[8],@x[9]
 604         eor     @x[10],@x[10],@x[11]
 605         eor     @x[12],@x[12],@x[13]
 606         eor     @x[14],@x[14],@x[15]
 607
 608         stp     @x[0],@x[2],[$out,#0]           // store output
 609          add    @d[6],@d[6],#4                  // increment counter
 610         stp     @x[4],@x[6],[$out,#16]
 611         stp     @x[8],@x[10],[$out,#32]
 612         stp     @x[12],@x[14],[$out,#48]
 613         add     $out,$out,#64
 614         b.eq    .Ldone_neon
 615         sub     $len,$len,#64
 616         cmp     $len,#64
 617         b.lo    .Less_than_128
 618
 619         ld1.8   {$T0-$T3},[$inp],#64
 620         eor     $A0,$A0,$T0
 621         eor     $B0,$B0,$T1
 622         eor     $C0,$C0,$T2
 623         eor     $D0,$D0,$T3
 624         st1.8   {$A0-$D0},[$out],#64
 625         b.eq    .Ldone_neon
 626         sub     $len,$len,#64
 627         cmp     $len,#64
 628         b.lo    .Less_than_192
 629
 630         ld1.8   {$T0-$T3},[$inp],#64
 631         eor     $A1,$A1,$T0
 632         eor     $B1,$B1,$T1
 633         eor     $C1,$C1,$T2
 634         eor     $D1,$D1,$T3
 635         st1.8   {$A1-$D1},[$out],#64
 636         b.eq    .Ldone_neon
 637         sub     $len,$len,#64
 638
 639         st1.8   {$A2-$D2},[sp]
 640         b       .Last_neon
 641
 642 .Less_than_128:
 643         st1.8   {$A0-$D0},[sp]
 644         b       .Last_neon
 645 .Less_than_192:
 646         st1.8   {$A1-$D1},[sp]
 647         b       .Last_neon
 648
 649 .align  4
 650 .Last_neon:
 651         sub     $out,$out,#1
 652         add     $inp,$inp,$len
 653         add     $out,$out,$len
 654         add     $ctr,sp,$len
 655         neg     $len,$len
 656
 657 .Loop_tail_neon:
 658         ldrb    w10,[$inp,$len]
 659         ldrb    w11,[$ctr,$len]
 660         add     $len,$len,#1
 661         eor     w10,w10,w11
 662         strb    w10,[$out,$len]
 663         cbnz    $len,.Loop_tail_neon
 664
 665         stp     xzr,xzr,[sp,#0]
 666         stp     xzr,xzr,[sp,#16]
 667         stp     xzr,xzr,[sp,#32]
 668         stp     xzr,xzr,[sp,#48]
 669
 670 .Ldone_neon:
 671         ldp     x19,x20,[x29,#16]
 672         add     sp,sp,#64
 673         ldp     x21,x22,[x29,#32]
 674         ldp     x23,x24,[x29,#48]
 675         ldp     x25,x26,[x29,#64]
 676         ldp     x27,x28,[x29,#80]
 677         ldp     x29,x30,[sp],#96
 678         ret
 679 .size   ChaCha20_neon,.-ChaCha20_neon
 680 ___
 681 {
 682 my ($T0,$T1,$T2,$T3,$T4,$T5)=@K;
 683 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,
 684     $A3,$B3,$C3,$D3,$A4,$B4,$C4,$D4,$A5,$B5,$C5,$D5) = map("v$_.4s",(0..23));
 685
 686 $code.=<<___;
 687 .type   ChaCha20_512_neon,%function
 688 .align  5
 689 ChaCha20_512_neon:
 690         stp     x29,x30,[sp,#-96]!
 691         add     x29,sp,#0
 692
 693         adr     @x[0],.Lsigma
 694         stp     x19,x20,[sp,#16]
 695         stp     x21,x22,[sp,#32]
 696         stp     x23,x24,[sp,#48]
 697         stp     x25,x26,[sp,#64]
 698         stp     x27,x28,[sp,#80]
 699
 700 .L512_or_more_neon:
 701         sub     sp,sp,#128+64
 702
 703         ldp     @d[0],@d[1],[@x[0]]             // load sigma
 704         ld1     {@K[0]},[@x[0]],#16
 705         ldp     @d[2],@d[3],[$key]              // load key
 706         ldp     @d[4],@d[5],[$key,#16]
 707         ld1     {@K[1],@K[2]},[$key]
 708         ldp     @d[6],@d[7],[$ctr]              // load counter
 709         ld1     {@K[3]},[$ctr]
 710         ld1     {$ONE},[@x[0]]
 711 #ifdef  __ARMEB__
 712         rev64   @K[0],@K[0]
 713         ror     @d[2],@d[2],#32
 714         ror     @d[3],@d[3],#32
 715         ror     @d[4],@d[4],#32
 716         ror     @d[5],@d[5],#32
 717         ror     @d[6],@d[6],#32
 718         ror     @d[7],@d[7],#32
 719 #endif
 720         add     @K[3],@K[3],$ONE                // += 1
 721         stp     @K[0],@K[1],[sp,#0]             // off-load key block, invariant part
 722         add     @K[3],@K[3],$ONE                // not typo
 723         str     @K[2],[sp,#32]
 724         add     @K[4],@K[3],$ONE
 725         add     @K[5],@K[4],$ONE
 726         add     @K[6],@K[5],$ONE
 727         shl     $ONE,$ONE,#2                    // 1 -> 4
 728
 729         stp     d8,d9,[sp,#128+0]               // meet ABI requirements
 730         stp     d10,d11,[sp,#128+16]
 731         stp     d12,d13,[sp,#128+32]
 732         stp     d14,d15,[sp,#128+48]
 733
 734         sub     $len,$len,#512                  // not typo
 735
 736 .Loop_outer_512_neon:
 737          mov    $A0,@K[0]
 738          mov    $A1,@K[0]
 739          mov    $A2,@K[0]
 740          mov    $A3,@K[0]
 741          mov    $A4,@K[0]
 742          mov    $A5,@K[0]
 743          mov    $B0,@K[1]
 744         mov.32  @x[0],@d[0]                     // unpack key block
 745          mov    $B1,@K[1]
 746         lsr     @x[1],@d[0],#32
 747          mov    $B2,@K[1]
 748         mov.32  @x[2],@d[1]
 749          mov    $B3,@K[1]
 750         lsr     @x[3],@d[1],#32
 751          mov    $B4,@K[1]
 752         mov.32  @x[4],@d[2]
 753          mov    $B5,@K[1]
 754         lsr     @x[5],@d[2],#32
 755          mov    $D0,@K[3]
 756         mov.32  @x[6],@d[3]
 757          mov    $D1,@K[4]
 758         lsr     @x[7],@d[3],#32
 759          mov    $D2,@K[5]
 760         mov.32  @x[8],@d[4]
 761          mov    $D3,@K[6]
 762         lsr     @x[9],@d[4],#32
 763          mov    $C0,@K[2]
 764         mov.32  @x[10],@d[5]
 765          mov    $C1,@K[2]
 766         lsr     @x[11],@d[5],#32
 767          add    $D4,$D0,$ONE                    // +4
 768         mov.32  @x[12],@d[6]
 769          add    $D5,$D1,$ONE                    // +4
 770         lsr     @x[13],@d[6],#32
 771          mov    $C2,@K[2]
 772         mov.32  @x[14],@d[7]
 773          mov    $C3,@K[2]
 774         lsr     @x[15],@d[7],#32
 775          mov    $C4,@K[2]
 776          stp    @K[3],@K[4],[sp,#48]            // off-load key block, variable part
 777          mov    $C5,@K[2]
 778          str    @K[5],[sp,#80]
 779
 780         mov     $ctr,#5
 781         subs    $len,$len,#512
 782 .Loop_upper_neon:
 783         sub     $ctr,$ctr,#1
 784 ___
 785         my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
 786         my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
 787         my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
 788         my @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0);
 789         my @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0);
 790         my @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0);
 791         my @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
 792         my $diff = ($#thread0+1)*6 - $#thread67 - 1;
 793         my $i = 0;
 794
 795         foreach (@thread0) {
 796                 eval;                   eval(shift(@thread67));
 797                 eval(shift(@thread1));  eval(shift(@thread67));
 798                 eval(shift(@thread2));  eval(shift(@thread67));
 799                 eval(shift(@thread3));  eval(shift(@thread67));
 800                 eval(shift(@thread4));  eval(shift(@thread67));
 801                 eval(shift(@thread5));  eval(shift(@thread67));
 802         }
 803
 804         @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
 805         @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
 806         @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
 807         @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1);
 808         @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1);
 809         @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1);
 810         @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
 811
 812         foreach (@thread0) {
 813                 eval;                   eval(shift(@thread67));
 814                 eval(shift(@thread1));  eval(shift(@thread67));
 815                 eval(shift(@thread2));  eval(shift(@thread67));
 816                 eval(shift(@thread3));  eval(shift(@thread67));
 817                 eval(shift(@thread4));  eval(shift(@thread67));
 818                 eval(shift(@thread5));  eval(shift(@thread67));
 819         }
 820 $code.=<<___;
 821         cbnz    $ctr,.Loop_upper_neon
 822
 823         add.32  @x[0],@x[0],@d[0]               // accumulate key block
 824         add     @x[1],@x[1],@d[0],lsr#32
 825         add.32  @x[2],@x[2],@d[1]
 826         add     @x[3],@x[3],@d[1],lsr#32
 827         add.32  @x[4],@x[4],@d[2]
 828         add     @x[5],@x[5],@d[2],lsr#32
 829         add.32  @x[6],@x[6],@d[3]
 830         add     @x[7],@x[7],@d[3],lsr#32
 831         add.32  @x[8],@x[8],@d[4]
 832         add     @x[9],@x[9],@d[4],lsr#32
 833         add.32  @x[10],@x[10],@d[5]
 834         add     @x[11],@x[11],@d[5],lsr#32
 835         add.32  @x[12],@x[12],@d[6]
 836         add     @x[13],@x[13],@d[6],lsr#32
 837         add.32  @x[14],@x[14],@d[7]
 838         add     @x[15],@x[15],@d[7],lsr#32
 839
 840         add     @x[0],@x[0],@x[1],lsl#32        // pack
 841         add     @x[2],@x[2],@x[3],lsl#32
 842         ldp     @x[1],@x[3],[$inp,#0]           // load input
 843         add     @x[4],@x[4],@x[5],lsl#32
 844         add     @x[6],@x[6],@x[7],lsl#32
 845         ldp     @x[5],@x[7],[$inp,#16]
 846         add     @x[8],@x[8],@x[9],lsl#32
 847         add     @x[10],@x[10],@x[11],lsl#32
 848         ldp     @x[9],@x[11],[$inp,#32]
 849         add     @x[12],@x[12],@x[13],lsl#32
 850         add     @x[14],@x[14],@x[15],lsl#32
 851         ldp     @x[13],@x[15],[$inp,#48]
 852         add     $inp,$inp,#64
 853 #ifdef  __ARMEB__
 854         rev     @x[0],@x[0]
 855         rev     @x[2],@x[2]
 856         rev     @x[4],@x[4]
 857         rev     @x[6],@x[6]
 858         rev     @x[8],@x[8]
 859         rev     @x[10],@x[10]
 860         rev     @x[12],@x[12]
 861         rev     @x[14],@x[14]
 862 #endif
 863         eor     @x[0],@x[0],@x[1]
 864         eor     @x[2],@x[2],@x[3]
 865         eor     @x[4],@x[4],@x[5]
 866         eor     @x[6],@x[6],@x[7]
 867         eor     @x[8],@x[8],@x[9]
 868         eor     @x[10],@x[10],@x[11]
 869         eor     @x[12],@x[12],@x[13]
 870         eor     @x[14],@x[14],@x[15]
 871
 872          stp    @x[0],@x[2],[$out,#0]           // store output
 873          add    @d[6],@d[6],#1                  // increment counter
 874         mov.32  @x[0],@d[0]                     // unpack key block
 875         lsr     @x[1],@d[0],#32
 876          stp    @x[4],@x[6],[$out,#16]
 877         mov.32  @x[2],@d[1]
 878         lsr     @x[3],@d[1],#32
 879          stp    @x[8],@x[10],[$out,#32]
 880         mov.32  @x[4],@d[2]
 881         lsr     @x[5],@d[2],#32
 882          stp    @x[12],@x[14],[$out,#48]
 883          add    $out,$out,#64
 884         mov.32  @x[6],@d[3]
 885         lsr     @x[7],@d[3],#32
 886         mov.32  @x[8],@d[4]
 887         lsr     @x[9],@d[4],#32
 888         mov.32  @x[10],@d[5]
 889         lsr     @x[11],@d[5],#32
 890         mov.32  @x[12],@d[6]
 891         lsr     @x[13],@d[6],#32
 892         mov.32  @x[14],@d[7]
 893         lsr     @x[15],@d[7],#32
 894
 895         mov     $ctr,#5
 896 .Loop_lower_neon:
 897         sub     $ctr,$ctr,#1
 898 ___
 899         @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
 900         @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
 901         @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
 902         @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0);
 903         @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0);
 904         @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0);
 905         @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
 906
 907         foreach (@thread0) {
 908                 eval;                   eval(shift(@thread67));
 909                 eval(shift(@thread1));  eval(shift(@thread67));
 910                 eval(shift(@thread2));  eval(shift(@thread67));
 911                 eval(shift(@thread3));  eval(shift(@thread67));
 912                 eval(shift(@thread4));  eval(shift(@thread67));
 913                 eval(shift(@thread5));  eval(shift(@thread67));
 914         }
 915
 916         @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
 917         @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
 918         @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
 919         @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1);
 920         @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1);
 921         @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1);
 922         @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
 923
 924         foreach (@thread0) {
 925                 eval;                   eval(shift(@thread67));
 926                 eval(shift(@thread1));  eval(shift(@thread67));
 927                 eval(shift(@thread2));  eval(shift(@thread67));
 928                 eval(shift(@thread3));  eval(shift(@thread67));
 929                 eval(shift(@thread4));  eval(shift(@thread67));
 930                 eval(shift(@thread5));  eval(shift(@thread67));
 931         }
 932 $code.=<<___;
 933         cbnz    $ctr,.Loop_lower_neon
 934
 935         add.32  @x[0],@x[0],@d[0]               // accumulate key block
 936          ldp    @K[0],@K[1],[sp,#0]
 937         add     @x[1],@x[1],@d[0],lsr#32
 938          ldp    @K[2],@K[3],[sp,#32]
 939         add.32  @x[2],@x[2],@d[1]
 940          ldp    @K[4],@K[5],[sp,#64]
 941         add     @x[3],@x[3],@d[1],lsr#32
 942          add    $A0,$A0,@K[0]
 943         add.32  @x[4],@x[4],@d[2]
 944          add    $A1,$A1,@K[0]
 945         add     @x[5],@x[5],@d[2],lsr#32
 946          add    $A2,$A2,@K[0]
 947         add.32  @x[6],@x[6],@d[3]
 948          add    $A3,$A3,@K[0]
 949         add     @x[7],@x[7],@d[3],lsr#32
 950          add    $A4,$A4,@K[0]
 951         add.32  @x[8],@x[8],@d[4]
 952          add    $A5,$A5,@K[0]
 953         add     @x[9],@x[9],@d[4],lsr#32
 954          add    $C0,$C0,@K[2]
 955         add.32  @x[10],@x[10],@d[5]
 956          add    $C1,$C1,@K[2]
 957         add     @x[11],@x[11],@d[5],lsr#32
 958          add    $C2,$C2,@K[2]
 959         add.32  @x[12],@x[12],@d[6]
 960          add    $C3,$C3,@K[2]
 961         add     @x[13],@x[13],@d[6],lsr#32
 962          add    $C4,$C4,@K[2]
 963         add.32  @x[14],@x[14],@d[7]
 964          add    $C5,$C5,@K[2]
 965         add     @x[15],@x[15],@d[7],lsr#32
 966          add    $D4,$D4,$ONE                    // +4
 967         add     @x[0],@x[0],@x[1],lsl#32        // pack
 968          add    $D5,$D5,$ONE                    // +4
 969         add     @x[2],@x[2],@x[3],lsl#32
 970          add    $D0,$D0,@K[3]
 971         ldp     @x[1],@x[3],[$inp,#0]           // load input
 972          add    $D1,$D1,@K[4]
 973         add     @x[4],@x[4],@x[5],lsl#32
 974          add    $D2,$D2,@K[5]
 975         add     @x[6],@x[6],@x[7],lsl#32
 976          add    $D3,$D3,@K[6]
 977         ldp     @x[5],@x[7],[$inp,#16]
 978          add    $D4,$D4,@K[3]
 979         add     @x[8],@x[8],@x[9],lsl#32
 980          add    $D5,$D5,@K[4]
 981         add     @x[10],@x[10],@x[11],lsl#32
 982          add    $B0,$B0,@K[1]
 983         ldp     @x[9],@x[11],[$inp,#32]
 984          add    $B1,$B1,@K[1]
 985         add     @x[12],@x[12],@x[13],lsl#32
 986          add    $B2,$B2,@K[1]
 987         add     @x[14],@x[14],@x[15],lsl#32
 988          add    $B3,$B3,@K[1]
 989         ldp     @x[13],@x[15],[$inp,#48]
 990          add    $B4,$B4,@K[1]
 991         add     $inp,$inp,#64
 992          add    $B5,$B5,@K[1]
 993
 994 #ifdef  __ARMEB__
 995         rev     @x[0],@x[0]
 996         rev     @x[2],@x[2]
 997         rev     @x[4],@x[4]
 998         rev     @x[6],@x[6]
 999         rev     @x[8],@x[8]
1000         rev     @x[10],@x[10]
1001         rev     @x[12],@x[12]
1002         rev     @x[14],@x[14]
1003 #endif
1004         ld1.8   {$T0-$T3},[$inp],#64
1005         eor     @x[0],@x[0],@x[1]
1006         eor     @x[2],@x[2],@x[3]
1007         eor     @x[4],@x[4],@x[5]
1008         eor     @x[6],@x[6],@x[7]
1009         eor     @x[8],@x[8],@x[9]
1010          eor    $A0,$A0,$T0
1011         eor     @x[10],@x[10],@x[11]
1012          eor    $B0,$B0,$T1
1013         eor     @x[12],@x[12],@x[13]
1014          eor    $C0,$C0,$T2
1015         eor     @x[14],@x[14],@x[15]
1016          eor    $D0,$D0,$T3
1017          ld1.8  {$T0-$T3},[$inp],#64
1018
1019         stp     @x[0],@x[2],[$out,#0]           // store output
1020          add    @d[6],@d[6],#7                  // increment counter
1021         stp     @x[4],@x[6],[$out,#16]
1022         stp     @x[8],@x[10],[$out,#32]
1023         stp     @x[12],@x[14],[$out,#48]
1024         add     $out,$out,#64
1025         st1.8   {$A0-$D0},[$out],#64
1026
1027         ld1.8   {$A0-$D0},[$inp],#64
1028         eor     $A1,$A1,$T0
1029         eor     $B1,$B1,$T1
1030         eor     $C1,$C1,$T2
1031         eor     $D1,$D1,$T3
1032         st1.8   {$A1-$D1},[$out],#64
1033
1034         ld1.8   {$A1-$D1},[$inp],#64
1035         eor     $A2,$A2,$A0
1036          ldp    @K[0],@K[1],[sp,#0]
1037         eor     $B2,$B2,$B0
1038          ldp    @K[2],@K[3],[sp,#32]
1039         eor     $C2,$C2,$C0
1040         eor     $D2,$D2,$D0
1041         st1.8   {$A2-$D2},[$out],#64
1042
1043         ld1.8   {$A2-$D2},[$inp],#64
1044         eor     $A3,$A3,$A1
1045         eor     $B3,$B3,$B1
1046         eor     $C3,$C3,$C1
1047         eor     $D3,$D3,$D1
1048         st1.8   {$A3-$D3},[$out],#64
1049
1050         ld1.8   {$A3-$D3},[$inp],#64
1051         eor     $A4,$A4,$A2
1052         eor     $B4,$B4,$B2
1053         eor     $C4,$C4,$C2
1054         eor     $D4,$D4,$D2
1055         st1.8   {$A4-$D4},[$out],#64
1056
1057         shl     $A0,$ONE,#1                     // 4 -> 8
1058         eor     $A5,$A5,$A3
1059         eor     $B5,$B5,$B3
1060         eor     $C5,$C5,$C3
1061         eor     $D5,$D5,$D3
1062         st1.8   {$A5-$D5},[$out],#64
1063
1064         add     @K[3],@K[3],$A0                 // += 8
1065         add     @K[4],@K[4],$A0
1066         add     @K[5],@K[5],$A0
1067         add     @K[6],@K[6],$A0
1068
1069         b.hs    .Loop_outer_512_neon
1070
1071         adds    $len,$len,#512
1072         ushr    $A0,$ONE,#2                     // 4 -> 1
1073
1074         ldp     d8,d9,[sp,#128+0]               // meet ABI requirements
1075         ldp     d10,d11,[sp,#128+16]
1076         ldp     d12,d13,[sp,#128+32]
1077         ldp     d14,d15,[sp,#128+48]
1078
1079         stp     @K[0],$ONE,[sp,#0]              // wipe off-load area
1080         stp     @K[0],$ONE,[sp,#32]
1081         stp     @K[0],$ONE,[sp,#64]
1082
1083         b.eq    .Ldone_512_neon
1084
1085         cmp     $len,#192
1086         sub     @K[3],@K[3],$A0                 // -= 1
1087         sub     @K[4],@K[4],$A0
1088         sub     @K[5],@K[5],$A0
1089         add     sp,sp,#128
1090         b.hs    .Loop_outer_neon
1091
1092         eor     @K[1],@K[1],@K[1]
1093         eor     @K[2],@K[2],@K[2]
1094         eor     @K[3],@K[3],@K[3]
1095         eor     @K[4],@K[4],@K[4]
1096         eor     @K[5],@K[5],@K[5]
1097         eor     @K[6],@K[6],@K[6]
1098         b       .Loop_outer
1099
1100 .Ldone_512_neon:
1101         ldp     x19,x20,[x29,#16]
1102         add     sp,sp,#128+64
1103         ldp     x21,x22,[x29,#32]
1104         ldp     x23,x24,[x29,#48]
1105         ldp     x25,x26,[x29,#64]
1106         ldp     x27,x28,[x29,#80]
1107         ldp     x29,x30,[sp],#96
1108         ret
1109 .size   ChaCha20_512_neon,.-ChaCha20_512_neon
1110 ___
1111 }
1112 }}}
1113
1114 foreach (split("\n",$code)) {
1115         s/\`([^\`]*)\`/eval $1/geo;
1116
1117         (s/\b([a-z]+)\.32\b/$1/ and (s/x([0-9]+)/w$1/g or 1))   or
1118         (m/\b(eor|ext|mov)\b/ and (s/\.4s/\.16b/g or 1))        or
1119         (s/\b((?:ld|st)1)\.8\b/$1/ and (s/\.4s/\.16b/g or 1))   or
1120         (m/\b(ld|st)[rp]\b/ and (s/v([0-9]+)\.4s/q$1/g or 1))   or
1121         (s/\brev32\.16\b/rev32/ and (s/\.4s/\.8h/g or 1));
1122
1123         #s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo;
1124
1125         print $_,"\n";
1126 }