chacha/asm/chacha-x86_64.pl: add AVX512VL code path.

256-bit AVX512VL was estimated to deliver ~50% improvement over AVX2
and it did live up to the expectations.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4838)

diff --git a/crypto/chacha/asm/chacha-x86_64.pl b/crypto/chacha/asm/chacha-x86_64.pl
index 0510363..51bb6a9 100755 (executable)
--- a/crypto/chacha/asm/chacha-x86_64.pl
+++ b/crypto/chacha/asm/chacha-x86_64.pl
@@ -22,6 +22,10 @@
 #
 # Add AVX512F code path.
 #
 #
 # Add AVX512F code path.
 #

+# December 2017
+#
+# Add AVX512VL code path.
+#

 # Performance in cycles per byte out of large buffer.
 #
 #              IALU/gcc 4.8(i) 1xSSSE3/SSE2    4xSSSE3     NxAVX(v)
 # Performance in cycles per byte out of large buffer.
 #
 #              IALU/gcc 4.8(i) 1xSSSE3/SSE2    4xSSSE3     NxAVX(v)


@@ -32,7 +36,7 @@
 # Sandy Bridge 8.31/+42%       5.45/6.76       2.72
 # Ivy Bridge   6.71/+46%       5.40/6.49       2.41
 # Haswell      5.92/+43%       5.20/6.45       2.42        1.23
 # Sandy Bridge 8.31/+42%       5.45/6.76       2.72
 # Ivy Bridge   6.71/+46%       5.40/6.49       2.41
 # Haswell      5.92/+43%       5.20/6.45       2.42        1.23

-# Skylake[-X]  5.87/+39%       4.70/-          2.31        1.19[0.57]
+# Skylake[-X]  5.87/+39%       4.70/-          2.31        1.19[0.80(vi)]

 # Silvermont   12.0/+33%       7.75/7.40       7.03(iii)
 # Knights L    11.7/-          -               9.60(iii)   0.80
 # Goldmont     10.6/+17%       5.10/-          3.28
 # Silvermont   12.0/+33%       7.75/7.40       7.03(iii)
 # Knights L    11.7/-          -               9.60(iii)   0.80
 # Goldmont     10.6/+17%       5.10/-          3.28


@@ -51,7 +55,9 @@
 #      limitations, SSE2 can do better, but gain is considered too
 #      low to justify the [maintenance] effort;
 # (iv) Bulldozer actually executes 4xXOP code path that delivers 2.20;
 #      limitations, SSE2 can do better, but gain is considered too
 #      low to justify the [maintenance] effort;
 # (iv) Bulldozer actually executes 4xXOP code path that delivers 2.20;

-# (v)  8xAVX2 or 16xAVX-512, whichever best applicable;
+# (v)  8xAVX2, 8xAVX512VL or 16xAVX512F, whichever best applicable;
+# (vi) even though Skylake-X can execute AVX512F code and deliver 0.57
+#      cpb in single thread, the corresponding capability is suppressed;

 
 $flavour = shift;
 $output  = shift;
 
 $flavour = shift;
 $output  = shift;


@@ -112,8 +118,8 @@ $code.=<<___;
 .byte  0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd
 .Lrot24:
 .byte  0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe
 .byte  0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd
 .Lrot24:
 .byte  0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe

-.Lsigma:
-.asciz "expand 32-byte k"
+.Ltwoy:
+.long  2,0,0,0, 2,0,0,0

 .align 64
 .Lzeroz:
 .long  0,0,0,0, 1,0,0,0, 2,0,0,0, 3,0,0,0
 .align 64
 .Lzeroz:
 .long  0,0,0,0, 1,0,0,0, 2,0,0,0, 3,0,0,0


@@ -123,6 +129,8 @@ $code.=<<___;
 .long  0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
 .Lsixteen:
 .long  16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16
 .long  0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
 .Lsixteen:
 .long  16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16

+.Lsigma:
+.asciz "expand 32-byte k"

 .asciz "ChaCha20 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
 ___
 
 .asciz "ChaCha20 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
 ___
 


@@ -253,6 +261,8 @@ ___
 $code.=<<___   if ($avx>2);
        bt      \$48,%r10               # check for AVX512F
        jc      .LChaCha20_avx512
 $code.=<<___   if ($avx>2);
        bt      \$48,%r10               # check for AVX512F
        jc      .LChaCha20_avx512

+       test    %r10,%r10               # check for AVX512VL
+       js      .LChaCha20_avx512vl

 ___
 $code.=<<___;
        test    \$`1<<(41-32)`,%r10d
 ___
 $code.=<<___;
        test    \$`1<<(41-32)`,%r10d


@@ -2292,6 +2302,19 @@ if ($avx>2) {
 my ($a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz) = map("%zmm$_",(0..3,16..20));
 my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7));
 
 my ($a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz) = map("%zmm$_",(0..3,16..20));
 my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7));
 

+sub vpxord()           # size optimization
+{ my $opcode = "vpxor";        # adhere to vpxor when possible
+
+    foreach (@_) {
+       if (/%([zy])mm([0-9]+)/ && ($1 eq "z" || $2>=16)) {
+           $opcode = "vpxord";
+           last;
+       }
+    }
+
+    $code .= "\t$opcode\t".join(',',reverse @_)."\n";
+}
+

 sub AVX512ROUND {      # critical path is 14 "SIMD ticks" per round
        &vpaddd ($a,$a,$b);
        &vpxord ($d,$d,$a);
 sub AVX512ROUND {      # critical path is 14 "SIMD ticks" per round
        &vpaddd ($a,$a,$b);
        &vpxord ($d,$d,$a);


@@ -2505,6 +2528,159 @@ $code.=<<___;
 .cfi_endproc
 .size  ChaCha20_avx512,.-ChaCha20_avx512
 ___
 .cfi_endproc
 .size  ChaCha20_avx512,.-ChaCha20_avx512
 ___

+
+map(s/%z/%y/, $a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz);
+
+$code.=<<___;
+.type  ChaCha20_avx512vl,\@function,5
+.align 32
+ChaCha20_avx512vl:
+.cfi_startproc
+.LChaCha20_avx512vl:
+       mov     %rsp,%r9                # frame pointer
+.cfi_def_cfa_register  %r9
+       cmp     \$128,$len
+       ja      .LChaCha20_8xvl
+
+       sub     \$64+$xframe,%rsp
+___
+$code.=<<___   if ($win64);
+       movaps  %xmm6,-0x28(%r9)
+       movaps  %xmm7,-0x18(%r9)
+.Lavx512vl_body:
+___
+$code.=<<___;
+       vbroadcasti128  .Lsigma(%rip),$a
+       vbroadcasti128  ($key),$b
+       vbroadcasti128  16($key),$c
+       vbroadcasti128  ($counter),$d
+
+       vmovdqa32       $a,$a_
+       vmovdqa32       $b,$b_
+       vmovdqa32       $c,$c_
+       vpaddd          .Lzeroz(%rip),$d,$d
+       vmovdqa32       .Ltwoy(%rip),$fourz
+       mov             \$10,$counter   # reuse $counter
+       vmovdqa32       $d,$d_
+       jmp             .Loop_avx512vl
+
+.align 16
+.Loop_outer_avx512vl:
+       vmovdqa32       $c_,$c
+       vpaddd          $fourz,$d_,$d
+       mov             \$10,$counter
+       vmovdqa32       $d,$d_
+       jmp             .Loop_avx512vl
+
+.align 32
+.Loop_avx512vl:
+___
+       &AVX512ROUND();
+       &vpshufd        ($c,$c,0b01001110);
+       &vpshufd        ($b,$b,0b00111001);
+       &vpshufd        ($d,$d,0b10010011);
+
+       &AVX512ROUND();
+       &vpshufd        ($c,$c,0b01001110);
+       &vpshufd        ($b,$b,0b10010011);
+       &vpshufd        ($d,$d,0b00111001);
+
+       &dec            ($counter);
+       &jnz            (".Loop_avx512vl");
+
+$code.=<<___;
+       vpaddd          $a_,$a,$a
+       vpaddd          $b_,$b,$b
+       vpaddd          $c_,$c,$c
+       vpaddd          $d_,$d,$d
+
+       sub             \$64,$len
+       jb              .Ltail64_avx512vl
+
+       vpxor           0x00($inp),%x#$a,$t0    # xor with input
+       vpxor           0x10($inp),%x#$b,$t1
+       vpxor           0x20($inp),%x#$c,$t2
+       vpxor           0x30($inp),%x#$d,$t3
+       lea             0x40($inp),$inp         # inp+=64
+
+       vmovdqu         $t0,0x00($out)          # write output
+       vmovdqu         $t1,0x10($out)
+       vmovdqu         $t2,0x20($out)
+       vmovdqu         $t3,0x30($out)
+       lea             0x40($out),$out         # out+=64
+
+       jz              .Ldone_avx512vl
+
+       vextracti128    \$1,$a,$t0
+       vextracti128    \$1,$b,$t1
+       vextracti128    \$1,$c,$t2
+       vextracti128    \$1,$d,$t3
+
+       sub             \$64,$len
+       jb              .Ltail_avx512vl
+
+       vpxor           0x00($inp),$t0,$t0      # xor with input
+       vpxor           0x10($inp),$t1,$t1
+       vpxor           0x20($inp),$t2,$t2
+       vpxor           0x30($inp),$t3,$t3
+       lea             0x40($inp),$inp         # inp+=64
+
+       vmovdqu         $t0,0x00($out)          # write output
+       vmovdqu         $t1,0x10($out)
+       vmovdqu         $t2,0x20($out)
+       vmovdqu         $t3,0x30($out)
+       lea             0x40($out),$out         # out+=64
+
+       vmovdqa32       $a_,$a
+       vmovdqa32       $b_,$b
+       jnz             .Loop_outer_avx512vl
+
+       jmp             .Ldone_avx512vl
+
+.align 16
+.Ltail64_avx512vl:
+       vmovdqa         %x#$a,0x00(%rsp)
+       vmovdqa         %x#$b,0x10(%rsp)
+       vmovdqa         %x#$c,0x20(%rsp)
+       vmovdqa         %x#$d,0x30(%rsp)
+       add             \$64,$len
+       jmp             .Loop_tail_avx512vl
+
+.align 16
+.Ltail_avx512vl:
+       vmovdqa         $t0,0x00(%rsp)
+       vmovdqa         $t1,0x10(%rsp)
+       vmovdqa         $t2,0x20(%rsp)
+       vmovdqa         $t3,0x30(%rsp)
+       add             \$64,$len
+
+.Loop_tail_avx512vl:
+       movzb           ($inp,$counter),%eax
+       movzb           (%rsp,$counter),%ecx
+       lea             1($counter),$counter
+       xor             %ecx,%eax
+       mov             %al,-1($out,$counter)
+       dec             $len
+       jnz             .Loop_tail_avx512vl
+
+       vmovdqu32       $a_,0x00(%rsp)
+       vmovdqu32       $a_,0x20(%rsp)
+
+.Ldone_avx512vl:
+       vzeroall
+___
+$code.=<<___   if ($win64);
+       movaps  -0x28(%r9),%xmm6
+       movaps  -0x18(%r9),%xmm7
+___
+$code.=<<___;
+       lea     (%r9),%rsp
+.cfi_def_cfa_register  %rsp
+.Lavx512vl_epilogue:
+       ret
+.cfi_endproc
+.size  ChaCha20_avx512vl,.-ChaCha20_avx512vl
+___

 }
 if ($avx>2) {
 # This one handles longer inputs...
 }
 if ($avx>2) {
 # This one handles longer inputs...


@@ -3011,6 +3187,396 @@ $code.=<<___;
 .cfi_endproc
 .size  ChaCha20_16x,.-ChaCha20_16x
 ___
 .cfi_endproc
 .size  ChaCha20_16x,.-ChaCha20_16x
 ___

+
+# switch to %ymm domain
+($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+ $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%ymm$_",(0..15));
+@xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
+     $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3);
+@key=map("%ymm$_",(16..31));
+($xt0,$xt1,$xt2,$xt3)=@key[0..3];
+
+$code.=<<___;
+.type  ChaCha20_8xvl,\@function,5
+.align 32
+ChaCha20_8xvl:
+.cfi_startproc
+.LChaCha20_8xvl:
+       mov             %rsp,%r9                # frame register
+.cfi_def_cfa_register  %r9
+       sub             \$64+$xframe,%rsp
+       and             \$-64,%rsp
+___
+$code.=<<___   if ($win64);
+       movaps          %xmm6,-0xa8(%r9)
+       movaps          %xmm7,-0x98(%r9)
+       movaps          %xmm8,-0x88(%r9)
+       movaps          %xmm9,-0x78(%r9)
+       movaps          %xmm10,-0x68(%r9)
+       movaps          %xmm11,-0x58(%r9)
+       movaps          %xmm12,-0x48(%r9)
+       movaps          %xmm13,-0x38(%r9)
+       movaps          %xmm14,-0x28(%r9)
+       movaps          %xmm15,-0x18(%r9)
+.L8xvl_body:
+___
+$code.=<<___;
+       vzeroupper
+
+       lea             .Lsigma(%rip),%r10
+       vbroadcasti128  (%r10),$xa3             # key[0]
+       vbroadcasti128  ($key),$xb3             # key[1]
+       vbroadcasti128  16($key),$xc3           # key[2]
+       vbroadcasti128  ($counter),$xd3         # key[3]
+
+       vpshufd         \$0x00,$xa3,$xa0        # smash key by lanes...
+       vpshufd         \$0x55,$xa3,$xa1
+       vpshufd         \$0xaa,$xa3,$xa2
+       vpshufd         \$0xff,$xa3,$xa3
+       vmovdqa64       $xa0,@key[0]
+       vmovdqa64       $xa1,@key[1]
+       vmovdqa64       $xa2,@key[2]
+       vmovdqa64       $xa3,@key[3]
+
+       vpshufd         \$0x00,$xb3,$xb0
+       vpshufd         \$0x55,$xb3,$xb1
+       vpshufd         \$0xaa,$xb3,$xb2
+       vpshufd         \$0xff,$xb3,$xb3
+       vmovdqa64       $xb0,@key[4]
+       vmovdqa64       $xb1,@key[5]
+       vmovdqa64       $xb2,@key[6]
+       vmovdqa64       $xb3,@key[7]
+
+       vpshufd         \$0x00,$xc3,$xc0
+       vpshufd         \$0x55,$xc3,$xc1
+       vpshufd         \$0xaa,$xc3,$xc2
+       vpshufd         \$0xff,$xc3,$xc3
+       vmovdqa64       $xc0,@key[8]
+       vmovdqa64       $xc1,@key[9]
+       vmovdqa64       $xc2,@key[10]
+       vmovdqa64       $xc3,@key[11]
+
+       vpshufd         \$0x00,$xd3,$xd0
+       vpshufd         \$0x55,$xd3,$xd1
+       vpshufd         \$0xaa,$xd3,$xd2
+       vpshufd         \$0xff,$xd3,$xd3
+       vpaddd          .Lincy(%rip),$xd0,$xd0  # don't save counters yet
+       vmovdqa64       $xd0,@key[12]
+       vmovdqa64       $xd1,@key[13]
+       vmovdqa64       $xd2,@key[14]
+       vmovdqa64       $xd3,@key[15]
+
+       mov             \$10,%eax
+       jmp             .Loop8xvl
+
+.align 32
+.Loop_outer8xvl:
+       #vpbroadcastd   0(%r10),$xa0            # reload key
+       #vpbroadcastd   4(%r10),$xa1
+       vpbroadcastd    8(%r10),$xa2
+       vpbroadcastd    12(%r10),$xa3
+       vpaddd          .Leight(%rip),@key[12],@key[12] # next SIMD counters
+       vmovdqa64       @key[4],$xb0
+       vmovdqa64       @key[5],$xb1
+       vmovdqa64       @key[6],$xb2
+       vmovdqa64       @key[7],$xb3
+       vmovdqa64       @key[8],$xc0
+       vmovdqa64       @key[9],$xc1
+       vmovdqa64       @key[10],$xc2
+       vmovdqa64       @key[11],$xc3
+       vmovdqa64       @key[12],$xd0
+       vmovdqa64       @key[13],$xd1
+       vmovdqa64       @key[14],$xd2
+       vmovdqa64       @key[15],$xd3
+
+       vmovdqa64       $xa0,@key[0]
+       vmovdqa64       $xa1,@key[1]
+       vmovdqa64       $xa2,@key[2]
+       vmovdqa64       $xa3,@key[3]
+
+       mov             \$10,%eax
+       jmp             .Loop8xvl
+
+.align 32
+.Loop8xvl:
+___
+       foreach (&AVX512_lane_ROUND(0, 4, 8,12)) { eval; }
+       foreach (&AVX512_lane_ROUND(0, 5,10,15)) { eval; }
+$code.=<<___;
+       dec             %eax
+       jnz             .Loop8xvl
+
+       vpaddd          @key[0],$xa0,$xa0       # accumulate key
+       vpaddd          @key[1],$xa1,$xa1
+       vpaddd          @key[2],$xa2,$xa2
+       vpaddd          @key[3],$xa3,$xa3
+
+       vpunpckldq      $xa1,$xa0,$xt2          # "de-interlace" data
+       vpunpckldq      $xa3,$xa2,$xt3
+       vpunpckhdq      $xa1,$xa0,$xa0
+       vpunpckhdq      $xa3,$xa2,$xa2
+       vpunpcklqdq     $xt3,$xt2,$xa1          # "a0"
+       vpunpckhqdq     $xt3,$xt2,$xt2          # "a1"
+       vpunpcklqdq     $xa2,$xa0,$xa3          # "a2"
+       vpunpckhqdq     $xa2,$xa0,$xa0          # "a3"
+___
+       ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2);
+$code.=<<___;
+       vpaddd          @key[4],$xb0,$xb0
+       vpaddd          @key[5],$xb1,$xb1
+       vpaddd          @key[6],$xb2,$xb2
+       vpaddd          @key[7],$xb3,$xb3
+
+       vpunpckldq      $xb1,$xb0,$xt2
+       vpunpckldq      $xb3,$xb2,$xt3
+       vpunpckhdq      $xb1,$xb0,$xb0
+       vpunpckhdq      $xb3,$xb2,$xb2
+       vpunpcklqdq     $xt3,$xt2,$xb1          # "b0"
+       vpunpckhqdq     $xt3,$xt2,$xt2          # "b1"
+       vpunpcklqdq     $xb2,$xb0,$xb3          # "b2"
+       vpunpckhqdq     $xb2,$xb0,$xb0          # "b3"
+___
+       ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2);
+$code.=<<___;
+       vshufi32x4      \$0,$xb0,$xa0,$xt3      # "de-interlace" further
+       vshufi32x4      \$3,$xb0,$xa0,$xb0
+       vshufi32x4      \$0,$xb1,$xa1,$xa0
+       vshufi32x4      \$3,$xb1,$xa1,$xb1
+       vshufi32x4      \$0,$xb2,$xa2,$xa1
+       vshufi32x4      \$3,$xb2,$xa2,$xb2
+       vshufi32x4      \$0,$xb3,$xa3,$xa2
+       vshufi32x4      \$3,$xb3,$xa3,$xb3
+___
+       ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3);
+$code.=<<___;
+       vpaddd          @key[8],$xc0,$xc0
+       vpaddd          @key[9],$xc1,$xc1
+       vpaddd          @key[10],$xc2,$xc2
+       vpaddd          @key[11],$xc3,$xc3
+
+       vpunpckldq      $xc1,$xc0,$xt2
+       vpunpckldq      $xc3,$xc2,$xt3
+       vpunpckhdq      $xc1,$xc0,$xc0
+       vpunpckhdq      $xc3,$xc2,$xc2
+       vpunpcklqdq     $xt3,$xt2,$xc1          # "c0"
+       vpunpckhqdq     $xt3,$xt2,$xt2          # "c1"
+       vpunpcklqdq     $xc2,$xc0,$xc3          # "c2"
+       vpunpckhqdq     $xc2,$xc0,$xc0          # "c3"
+___
+       ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2);
+$code.=<<___;
+       vpaddd          @key[12],$xd0,$xd0
+       vpaddd          @key[13],$xd1,$xd1
+       vpaddd          @key[14],$xd2,$xd2
+       vpaddd          @key[15],$xd3,$xd3
+
+       vpunpckldq      $xd1,$xd0,$xt2
+       vpunpckldq      $xd3,$xd2,$xt3
+       vpunpckhdq      $xd1,$xd0,$xd0
+       vpunpckhdq      $xd3,$xd2,$xd2
+       vpunpcklqdq     $xt3,$xt2,$xd1          # "d0"
+       vpunpckhqdq     $xt3,$xt2,$xt2          # "d1"
+       vpunpcklqdq     $xd2,$xd0,$xd3          # "d2"
+       vpunpckhqdq     $xd2,$xd0,$xd0          # "d3"
+___
+       ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2);
+$code.=<<___;
+       vperm2i128      \$0x20,$xd0,$xc0,$xt3   # "de-interlace" further
+       vperm2i128      \$0x31,$xd0,$xc0,$xd0
+       vperm2i128      \$0x20,$xd1,$xc1,$xc0
+       vperm2i128      \$0x31,$xd1,$xc1,$xd1
+       vperm2i128      \$0x20,$xd2,$xc2,$xc1
+       vperm2i128      \$0x31,$xd2,$xc2,$xd2
+       vperm2i128      \$0x20,$xd3,$xc3,$xc2
+       vperm2i128      \$0x31,$xd3,$xc3,$xd3
+___
+       ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3);
+       ($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)=
+       ($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3);
+$code.=<<___;
+       cmp             \$64*8,$len
+       jb              .Ltail8xvl
+
+       mov             \$0x80,%eax             # size optimization
+       vpxord          0x00($inp),$xa0,$xa0    # xor with input
+       vpxor           0x20($inp),$xb0,$xb0
+       vpxor           0x40($inp),$xc0,$xc0
+       vpxor           0x60($inp),$xd0,$xd0
+       lea             ($inp,%rax),$inp        # size optimization
+       vmovdqu32       $xa0,0x00($out)
+       vmovdqu         $xb0,0x20($out)
+       vmovdqu         $xc0,0x40($out)
+       vmovdqu         $xd0,0x60($out)
+       lea             ($out,%rax),$out        # size optimization
+
+       vpxor           0x00($inp),$xa1,$xa1
+       vpxor           0x20($inp),$xb1,$xb1
+       vpxor           0x40($inp),$xc1,$xc1
+       vpxor           0x60($inp),$xd1,$xd1
+       lea             ($inp,%rax),$inp        # size optimization
+       vmovdqu         $xa1,0x00($out)
+       vmovdqu         $xb1,0x20($out)
+       vmovdqu         $xc1,0x40($out)
+       vmovdqu         $xd1,0x60($out)
+       lea             ($out,%rax),$out        # size optimization
+
+       vpxord          0x00($inp),$xa2,$xa2
+       vpxor           0x20($inp),$xb2,$xb2
+       vpxor           0x40($inp),$xc2,$xc2
+       vpxor           0x60($inp),$xd2,$xd2
+       lea             ($inp,%rax),$inp        # size optimization
+       vmovdqu32       $xa2,0x00($out)
+       vmovdqu         $xb2,0x20($out)
+       vmovdqu         $xc2,0x40($out)
+       vmovdqu         $xd2,0x60($out)
+       lea             ($out,%rax),$out        # size optimization
+
+       vpxor           0x00($inp),$xa3,$xa3
+       vpxor           0x20($inp),$xb3,$xb3
+       vpxor           0x40($inp),$xc3,$xc3
+       vpxor           0x60($inp),$xd3,$xd3
+       lea             ($inp,%rax),$inp        # size optimization
+       vmovdqu         $xa3,0x00($out)
+       vmovdqu         $xb3,0x20($out)
+       vmovdqu         $xc3,0x40($out)
+       vmovdqu         $xd3,0x60($out)
+       lea             ($out,%rax),$out        # size optimization
+
+       vpbroadcastd    0(%r10),%ymm0           # reload key
+       vpbroadcastd    4(%r10),%ymm1
+
+       sub             \$64*8,$len
+       jnz             .Loop_outer8xvl
+
+       jmp             .Ldone8xvl
+
+.align 32
+.Ltail8xvl:
+       vmovdqa64       $xa0,%ymm8              # size optimization
+___
+$xa0 = "%ymm8";
+$code.=<<___;
+       xor             %r10,%r10
+       sub             $inp,$out
+       cmp             \$64*1,$len
+       jb              .Less_than_64_8xvl
+       vpxor           0x00($inp),$xa0,$xa0    # xor with input
+       vpxor           0x20($inp),$xb0,$xb0
+       vmovdqu         $xa0,0x00($out,$inp)
+       vmovdqu         $xb0,0x20($out,$inp)
+       je              .Ldone8xvl
+       vmovdqa         $xc0,$xa0
+       vmovdqa         $xd0,$xb0
+       lea             64($inp),$inp
+
+       cmp             \$64*2,$len
+       jb              .Less_than_64_8xvl
+       vpxor           0x00($inp),$xc0,$xc0
+       vpxor           0x20($inp),$xd0,$xd0
+       vmovdqu         $xc0,0x00($out,$inp)
+       vmovdqu         $xd0,0x20($out,$inp)
+       je              .Ldone8xvl
+       vmovdqa         $xa1,$xa0
+       vmovdqa         $xb1,$xb0
+       lea             64($inp),$inp
+
+       cmp             \$64*3,$len
+       jb              .Less_than_64_8xvl
+       vpxor           0x00($inp),$xa1,$xa1
+       vpxor           0x20($inp),$xb1,$xb1
+       vmovdqu         $xa1,0x00($out,$inp)
+       vmovdqu         $xb1,0x20($out,$inp)
+       je              .Ldone8xvl
+       vmovdqa         $xc1,$xa0
+       vmovdqa         $xd1,$xb0
+       lea             64($inp),$inp
+
+       cmp             \$64*4,$len
+       jb              .Less_than_64_8xvl
+       vpxor           0x00($inp),$xc1,$xc1
+       vpxor           0x20($inp),$xd1,$xd1
+       vmovdqu         $xc1,0x00($out,$inp)
+       vmovdqu         $xd1,0x20($out,$inp)
+       je              .Ldone8xvl
+       vmovdqa32       $xa2,$xa0
+       vmovdqa         $xb2,$xb0
+       lea             64($inp),$inp
+
+       cmp             \$64*5,$len
+       jb              .Less_than_64_8xvl
+       vpxord          0x00($inp),$xa2,$xa2
+       vpxor           0x20($inp),$xb2,$xb2
+       vmovdqu32       $xa2,0x00($out,$inp)
+       vmovdqu         $xb2,0x20($out,$inp)
+       je              .Ldone8xvl
+       vmovdqa         $xc2,$xa0
+       vmovdqa         $xd2,$xb0
+       lea             64($inp),$inp
+
+       cmp             \$64*6,$len
+       jb              .Less_than_64_8xvl
+       vpxor           0x00($inp),$xc2,$xc2
+       vpxor           0x20($inp),$xd2,$xd2
+       vmovdqu         $xc2,0x00($out,$inp)
+       vmovdqu         $xd2,0x20($out,$inp)
+       je              .Ldone8xvl
+       vmovdqa         $xa3,$xa0
+       vmovdqa         $xb3,$xb0
+       lea             64($inp),$inp
+
+       cmp             \$64*7,$len
+       jb              .Less_than_64_8xvl
+       vpxor           0x00($inp),$xa3,$xa3
+       vpxor           0x20($inp),$xb3,$xb3
+       vmovdqu         $xa3,0x00($out,$inp)
+       vmovdqu         $xb3,0x20($out,$inp)
+       je              .Ldone8xvl
+       vmovdqa         $xc3,$xa0
+       vmovdqa         $xd3,$xb0
+       lea             64($inp),$inp
+
+.Less_than_64_8xvl:
+       vmovdqa         $xa0,0x00(%rsp)
+       vmovdqa         $xb0,0x20(%rsp)
+       lea             ($out,$inp),$out
+       and             \$63,$len
+
+.Loop_tail8xvl:
+       movzb           ($inp,%r10),%eax
+       movzb           (%rsp,%r10),%ecx
+       lea             1(%r10),%r10
+       xor             %ecx,%eax
+       mov             %al,-1($out,%r10)
+       dec             $len
+       jnz             .Loop_tail8xvl
+
+       vpxor           $xa0,$xa0,$xa0
+       vmovdqa         $xa0,0x00(%rsp)
+       vmovdqa         $xa0,0x20(%rsp)
+
+.Ldone8xvl:
+       vzeroall
+___
+$code.=<<___   if ($win64);
+       movaps          -0xa8(%r9),%xmm6
+       movaps          -0x98(%r9),%xmm7
+       movaps          -0x88(%r9),%xmm8
+       movaps          -0x78(%r9),%xmm9
+       movaps          -0x68(%r9),%xmm10
+       movaps          -0x58(%r9),%xmm11
+       movaps          -0x48(%r9),%xmm12
+       movaps          -0x38(%r9),%xmm13
+       movaps          -0x28(%r9),%xmm14
+       movaps          -0x18(%r9),%xmm15
+___
+$code.=<<___;
+       lea             (%r9),%rsp
+.cfi_def_cfa_register  %rsp
+.L8xvl_epilogue:
+       ret
+.cfi_endproc
+.size  ChaCha20_8xvl,.-ChaCha20_8xvl
+___

 }
 
 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
 }
 
 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,


@@ -3217,9 +3783,17 @@ $code.=<<___ if ($avx>2);
        .rva    .LSEH_end_ChaCha20_avx512
        .rva    .LSEH_info_ChaCha20_avx512
 
        .rva    .LSEH_end_ChaCha20_avx512
        .rva    .LSEH_info_ChaCha20_avx512
 

+       .rva    .LSEH_begin_ChaCha20_avx512vl
+       .rva    .LSEH_end_ChaCha20_avx512vl
+       .rva    .LSEH_info_ChaCha20_avx512vl
+

        .rva    .LSEH_begin_ChaCha20_16x
        .rva    .LSEH_end_ChaCha20_16x
        .rva    .LSEH_info_ChaCha20_16x
        .rva    .LSEH_begin_ChaCha20_16x
        .rva    .LSEH_end_ChaCha20_16x
        .rva    .LSEH_info_ChaCha20_16x

+
+       .rva    .LSEH_begin_ChaCha20_8xvl
+       .rva    .LSEH_end_ChaCha20_8xvl
+       .rva    .LSEH_info_ChaCha20_8xvl

 ___
 $code.=<<___;
 .section       .xdata
 ___
 $code.=<<___;
 .section       .xdata


@@ -3256,10 +3830,20 @@ $code.=<<___ if ($avx>2);
        .rva    ssse3_handler
        .rva    .Lavx512_body,.Lavx512_epilogue         # HandlerData[]
 
        .rva    ssse3_handler
        .rva    .Lavx512_body,.Lavx512_epilogue         # HandlerData[]
 

+.LSEH_info_ChaCha20_avx512vl:
+       .byte   9,0,0,0
+       .rva    ssse3_handler
+       .rva    .Lavx512vl_body,.Lavx512vl_epilogue     # HandlerData[]
+

 .LSEH_info_ChaCha20_16x:
        .byte   9,0,0,0
        .rva    full_handler
        .rva    .L16x_body,.L16x_epilogue               # HandlerData[]
 .LSEH_info_ChaCha20_16x:
        .byte   9,0,0,0
        .rva    full_handler
        .rva    .L16x_body,.L16x_epilogue               # HandlerData[]

+
+.LSEH_info_ChaCha20_8xvl:
+       .byte   9,0,0,0
+       .rva    full_handler
+       .rva    .L8xvl_body,.L8xvl_epilogue             # HandlerData[]

 ___
 }
 
 ___
 }