+ {{{
+my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10));
+my @MSG0=map("%xmm$_",(4..7));
+my @MSG1=map("%xmm$_",(11..14));
+
+$code.=<<___;
+.type sha1_multi_block_shaext,\@function,3
+.align 32
+sha1_multi_block_shaext:
+.cfi_startproc
+_shaext_shortcut:
+ mov %rsp,%rax
+.cfi_def_cfa_register %rax
+ push %rbx
+.cfi_push %rbx
+ push %rbp
+.cfi_push %rbp
+___
+$code.=<<___ if ($win64);
+ lea -0xa8(%rsp),%rsp
+ movaps %xmm6,(%rsp)
+ movaps %xmm7,0x10(%rsp)
+ movaps %xmm8,0x20(%rsp)
+ movaps %xmm9,0x30(%rsp)
+ movaps %xmm10,-0x78(%rax)
+ movaps %xmm11,-0x68(%rax)
+ movaps %xmm12,-0x58(%rax)
+ movaps %xmm13,-0x48(%rax)
+ movaps %xmm14,-0x38(%rax)
+ movaps %xmm15,-0x28(%rax)
+___
+$code.=<<___;
+ sub \$`$REG_SZ*18`,%rsp
+ shl \$1,$num # we process pair at a time
+ and \$-256,%rsp
+ lea 0x40($ctx),$ctx # size optimization
+ mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
+.Lbody_shaext:
+ lea `$REG_SZ*16`(%rsp),%rbx
+ movdqa K_XX_XX+0x80(%rip),$BSWAP # byte-n-word swap
+
+.Loop_grande_shaext:
+ mov $num,`$REG_SZ*17+8`(%rsp) # original $num
+ xor $num,$num
+___
+for($i=0;$i<2;$i++) {
+ $code.=<<___;
+ mov `16*$i+0`($inp),@ptr[$i] # input pointer
+ mov `16*$i+8`($inp),%ecx # number of blocks
+ cmp $num,%ecx
+ cmovg %ecx,$num # find maximum
+ test %ecx,%ecx
+ mov %ecx,`4*$i`(%rbx) # initialize counters
+ cmovle %rsp,@ptr[$i] # cancel input
+___
+}
+$code.=<<___;
+ test $num,$num
+ jz .Ldone_shaext
+
+ movq 0x00-0x40($ctx),$ABCD0 # a1.a0
+ movq 0x20-0x40($ctx),@MSG0[0]# b1.b0
+ movq 0x40-0x40($ctx),@MSG0[1]# c1.c0
+ movq 0x60-0x40($ctx),@MSG0[2]# d1.d0
+ movq 0x80-0x40($ctx),@MSG0[3]# e1.e0
+
+ punpckldq @MSG0[0],$ABCD0 # b1.a1.b0.a0
+ punpckldq @MSG0[2],@MSG0[1] # d1.c1.d0.c0
+
+ movdqa $ABCD0,$ABCD1
+ punpcklqdq @MSG0[1],$ABCD0 # d0.c0.b0.a0
+ punpckhqdq @MSG0[1],$ABCD1 # d1.c1.b1.a1
+
+ pshufd \$0b00111111,@MSG0[3],$E0
+ pshufd \$0b01111111,@MSG0[3],$E1
+ pshufd \$0b00011011,$ABCD0,$ABCD0
+ pshufd \$0b00011011,$ABCD1,$ABCD1
+ jmp .Loop_shaext
+
+.align 32
+.Loop_shaext:
+ movdqu 0x00(@ptr[0]),@MSG0[0]
+ movdqu 0x00(@ptr[1]),@MSG1[0]
+ movdqu 0x10(@ptr[0]),@MSG0[1]
+ movdqu 0x10(@ptr[1]),@MSG1[1]
+ movdqu 0x20(@ptr[0]),@MSG0[2]
+ pshufb $BSWAP,@MSG0[0]
+ movdqu 0x20(@ptr[1]),@MSG1[2]
+ pshufb $BSWAP,@MSG1[0]
+ movdqu 0x30(@ptr[0]),@MSG0[3]
+ lea 0x40(@ptr[0]),@ptr[0]
+ pshufb $BSWAP,@MSG0[1]
+ movdqu 0x30(@ptr[1]),@MSG1[3]
+ lea 0x40(@ptr[1]),@ptr[1]
+ pshufb $BSWAP,@MSG1[1]
+
+ movdqa $E0,0x50(%rsp) # offload
+ paddd @MSG0[0],$E0
+ movdqa $E1,0x70(%rsp)
+ paddd @MSG1[0],$E1
+ movdqa $ABCD0,0x40(%rsp) # offload
+ movdqa $ABCD0,$E0_
+ movdqa $ABCD1,0x60(%rsp)
+ movdqa $ABCD1,$E1_
+ sha1rnds4 \$0,$E0,$ABCD0 # 0-3
+ sha1nexte @MSG0[1],$E0_
+ sha1rnds4 \$0,$E1,$ABCD1 # 0-3
+ sha1nexte @MSG1[1],$E1_
+ pshufb $BSWAP,@MSG0[2]
+ prefetcht0 127(@ptr[0])
+ sha1msg1 @MSG0[1],@MSG0[0]
+ pshufb $BSWAP,@MSG1[2]
+ prefetcht0 127(@ptr[1])
+ sha1msg1 @MSG1[1],@MSG1[0]
+
+ pshufb $BSWAP,@MSG0[3]
+ movdqa $ABCD0,$E0
+ pshufb $BSWAP,@MSG1[3]
+ movdqa $ABCD1,$E1
+ sha1rnds4 \$0,$E0_,$ABCD0 # 4-7
+ sha1nexte @MSG0[2],$E0
+ sha1rnds4 \$0,$E1_,$ABCD1 # 4-7
+ sha1nexte @MSG1[2],$E1
+ pxor @MSG0[2],@MSG0[0]
+ sha1msg1 @MSG0[2],@MSG0[1]
+ pxor @MSG1[2],@MSG1[0]
+ sha1msg1 @MSG1[2],@MSG1[1]
+___
+for($i=2;$i<20-4;$i++) {
+$code.=<<___;
+ movdqa $ABCD0,$E0_
+ movdqa $ABCD1,$E1_
+ sha1rnds4 \$`int($i/5)`,$E0,$ABCD0 # 8-11
+ sha1nexte @MSG0[3],$E0_
+ sha1rnds4 \$`int($i/5)`,$E1,$ABCD1 # 8-11
+ sha1nexte @MSG1[3],$E1_
+ sha1msg2 @MSG0[3],@MSG0[0]
+ sha1msg2 @MSG1[3],@MSG1[0]
+ pxor @MSG0[3],@MSG0[1]
+ sha1msg1 @MSG0[3],@MSG0[2]
+ pxor @MSG1[3],@MSG1[1]
+ sha1msg1 @MSG1[3],@MSG1[2]
+___
+ ($E0,$E0_)=($E0_,$E0); ($E1,$E1_)=($E1_,$E1);
+ push(@MSG0,shift(@MSG0)); push(@MSG1,shift(@MSG1));
+}
+$code.=<<___;
+ movdqa $ABCD0,$E0_
+ movdqa $ABCD1,$E1_
+ sha1rnds4 \$3,$E0,$ABCD0 # 64-67
+ sha1nexte @MSG0[3],$E0_
+ sha1rnds4 \$3,$E1,$ABCD1 # 64-67
+ sha1nexte @MSG1[3],$E1_
+ sha1msg2 @MSG0[3],@MSG0[0]
+ sha1msg2 @MSG1[3],@MSG1[0]
+ pxor @MSG0[3],@MSG0[1]
+ pxor @MSG1[3],@MSG1[1]
+
+ mov \$1,%ecx
+ pxor @MSG0[2],@MSG0[2] # zero
+ cmp 4*0(%rbx),%ecx # examine counters
+ cmovge %rsp,@ptr[0] # cancel input
+
+ movdqa $ABCD0,$E0
+ movdqa $ABCD1,$E1
+ sha1rnds4 \$3,$E0_,$ABCD0 # 68-71
+ sha1nexte @MSG0[0],$E0
+ sha1rnds4 \$3,$E1_,$ABCD1 # 68-71
+ sha1nexte @MSG1[0],$E1
+ sha1msg2 @MSG0[0],@MSG0[1]
+ sha1msg2 @MSG1[0],@MSG1[1]
+
+ cmp 4*1(%rbx),%ecx
+ cmovge %rsp,@ptr[1]
+ movq (%rbx),@MSG0[0] # pull counters
+
+ movdqa $ABCD0,$E0_
+ movdqa $ABCD1,$E1_
+ sha1rnds4 \$3,$E0,$ABCD0 # 72-75
+ sha1nexte @MSG0[1],$E0_
+ sha1rnds4 \$3,$E1,$ABCD1 # 72-75
+ sha1nexte @MSG1[1],$E1_
+
+ pshufd \$0x00,@MSG0[0],@MSG1[2]
+ pshufd \$0x55,@MSG0[0],@MSG1[3]
+ movdqa @MSG0[0],@MSG0[1]
+ pcmpgtd @MSG0[2],@MSG1[2]
+ pcmpgtd @MSG0[2],@MSG1[3]
+
+ movdqa $ABCD0,$E0
+ movdqa $ABCD1,$E1
+ sha1rnds4 \$3,$E0_,$ABCD0 # 76-79
+ sha1nexte $MSG0[2],$E0
+ sha1rnds4 \$3,$E1_,$ABCD1 # 76-79
+ sha1nexte $MSG0[2],$E1
+
+ pcmpgtd @MSG0[2],@MSG0[1] # counter mask
+ pand @MSG1[2],$ABCD0
+ pand @MSG1[2],$E0
+ pand @MSG1[3],$ABCD1
+ pand @MSG1[3],$E1
+ paddd @MSG0[1],@MSG0[0] # counters--
+
+ paddd 0x40(%rsp),$ABCD0
+ paddd 0x50(%rsp),$E0
+ paddd 0x60(%rsp),$ABCD1
+ paddd 0x70(%rsp),$E1
+
+ movq @MSG0[0],(%rbx) # save counters
+ dec $num
+ jnz .Loop_shaext
+
+ mov `$REG_SZ*17+8`(%rsp),$num
+
+ pshufd \$0b00011011,$ABCD0,$ABCD0
+ pshufd \$0b00011011,$ABCD1,$ABCD1
+
+ movdqa $ABCD0,@MSG0[0]
+ punpckldq $ABCD1,$ABCD0 # b1.b0.a1.a0
+ punpckhdq $ABCD1,@MSG0[0] # d1.d0.c1.c0
+ punpckhdq $E1,$E0 # e1.e0.xx.xx
+ movq $ABCD0,0x00-0x40($ctx) # a1.a0
+ psrldq \$8,$ABCD0
+ movq @MSG0[0],0x40-0x40($ctx)# c1.c0
+ psrldq \$8,@MSG0[0]
+ movq $ABCD0,0x20-0x40($ctx) # b1.b0
+ psrldq \$8,$E0
+ movq @MSG0[0],0x60-0x40($ctx)# d1.d0
+ movq $E0,0x80-0x40($ctx) # e1.e0
+
+ lea `$REG_SZ/2`($ctx),$ctx
+ lea `16*2`($inp),$inp
+ dec $num
+ jnz .Loop_grande_shaext
+
+.Ldone_shaext:
+ #mov `$REG_SZ*17`(%rsp),%rax # original %rsp
+___
+$code.=<<___ if ($win64);
+ movaps -0xb8(%rax),%xmm6
+ movaps -0xa8(%rax),%xmm7
+ movaps -0x98(%rax),%xmm8
+ movaps -0x88(%rax),%xmm9
+ movaps -0x78(%rax),%xmm10
+ movaps -0x68(%rax),%xmm11
+ movaps -0x58(%rax),%xmm12
+ movaps -0x48(%rax),%xmm13
+ movaps -0x38(%rax),%xmm14
+ movaps -0x28(%rax),%xmm15
+___
+$code.=<<___;
+ mov -16(%rax),%rbp
+.cfi_restore %rbp
+ mov -8(%rax),%rbx
+.cfi_restore %rbx
+ lea (%rax),%rsp
+.cfi_def_cfa_register %rsp
+.Lepilogue_shaext:
+ ret
+.cfi_endproc
+.size sha1_multi_block_shaext,.-sha1_multi_block_shaext
+___
+ }}}