crypto/sha/asm/sha1-mb-x86_64.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9
  10 # Multi-buffer SHA1 procedure processes n buffers in parallel by
  11 # placing buffer data to designated lane of SIMD register. n is
  12 # naturally limited to 4 on pre-AVX2 processors and to 8 on
  13 # AVX2-capable processors such as Haswell.
  14 #
  15 #               this    +aesni(i)       sha1    aesni-sha1      gain(iv)
  16 # -------------------------------------------------------------------
  17 # Westmere(ii)  10.7/n  +1.28=3.96(n=4) 5.30    6.66            +68%
  18 # Atom(ii)      18.1/n  +3.93=8.46(n=4) 9.37    12.8            +51%
  19 # Sandy Bridge  (8.16   +5.15=13.3)/n   4.99    5.98            +80%
  20 # Ivy Bridge    (8.08   +5.14=13.2)/n   4.60    5.54            +68%
  21 # Haswell(iii)  (8.96   +5.00=14.0)/n   3.57    4.55            +160%
  22 # Bulldozer     (9.76   +5.76=15.5)/n   5.95    6.37            +64%
  23 #
  24 # (i)   multi-block CBC encrypt with 128-bit key;
  25 # (ii)  (HASH+AES)/n does not apply to Westmere for n>3 and Atom,
  26 #       because of lower AES-NI instruction throughput;
  27 # (iii) "this" is for n=8, when we gather twice as much data, result
  28 #       for n=4 is 8.00+4.44=12.4;
  29 # (iv)  presented improvement coefficients are asymptotic limits and
  30 #       in real-life application are somewhat lower, e.g. for 2KB
  31 #       fragments they range from 30% to 100% (on Haswell);
  32
  33 $flavour = shift;
  34 $output  = shift;
  35 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  36
  37 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  38
  39 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  40 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  41 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  42 die "can't locate x86_64-xlate.pl";
  43
  44 $avx=0;
  45
  46 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  47                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  48         $avx = ($1>=2.19) + ($1>=2.22);
  49 }
  50
  51 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  52            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  53         $avx = ($1>=2.09) + ($1>=2.10);
  54 }
  55
  56 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  57            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  58         $avx = ($1>=10) + ($1>=11);
  59 }
  60
  61 if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) {
  62         $avx = ($2>=3.0) + ($2>3.0);
  63 }
  64
  65 open OUT,"| \"$^X\" $xlate $flavour $output";
  66 *STDOUT=*OUT;
  67
  68 # void sha1_multi_block (
  69 #     struct {  unsigned int A[8];
  70 #               unsigned int B[8];
  71 #               unsigned int C[8];
  72 #               unsigned int D[8];
  73 #               unsigned int E[8];      } *ctx,
  74 #     struct {  void *ptr; int blocks;  } inp[8],
  75 #     int num);         /* 1 or 2 */
  76 #
  77 $ctx="%rdi";    # 1st arg
  78 $inp="%rsi";    # 2nd arg
  79 $num="%edx";
  80 @ptr=map("%r$_",(8..11));
  81 $Tbl="%rbp";
  82
  83 @V=($A,$B,$C,$D,$E)=map("%xmm$_",(0..4));
  84 ($t0,$t1,$t2,$t3,$tx)=map("%xmm$_",(5..9));
  85 @Xi=map("%xmm$_",(10..14));
  86 $K="%xmm15";
  87
  88 if (1) {
  89     # Atom-specific optimization aiming to eliminate pshufb with high
  90     # registers [and thus get rid of 48 cycles accumulated penalty]
  91     @Xi=map("%xmm$_",(0..4));
  92     ($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9));
  93     @V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14));
  94 }
  95
  96 $REG_SZ=16;
  97
  98 sub Xi_off {
  99 my $off = shift;
 100
 101     $off %= 16; $off *= $REG_SZ;
 102     $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
 103 }
 104
 105 sub BODY_00_19 {
 106 my ($i,$a,$b,$c,$d,$e)=@_;
 107 my $j=$i+1;
 108 my $k=$i+2;
 109
 110 # Loads are performed 2+3/4 iterations in advance. 3/4 means that out
 111 # of 4 words you would expect to be loaded per given iteration one is
 112 # spilled to next iteration. In other words indices in four input
 113 # streams are distributed as following:
 114 #
 115 # $i==0:        0,0,0,0,1,1,1,1,2,2,2,
 116 # $i==1:        2,3,3,3,
 117 # $i==2:        3,4,4,4,
 118 # ...
 119 # $i==13:       14,15,15,15,
 120 # $i==14:       15
 121 #
 122 # Then at $i==15 Xupdate is applied one iteration in advance...
 123 $code.=<<___ if ($i==0);
 124         movd            (@ptr[0]),@Xi[0]
 125          lea            `16*4`(@ptr[0]),@ptr[0]
 126         movd            (@ptr[1]),@Xi[2]        # borrow @Xi[2]
 127          lea            `16*4`(@ptr[1]),@ptr[1]
 128         movd            (@ptr[2]),@Xi[3]        # borrow @Xi[3]
 129          lea            `16*4`(@ptr[2]),@ptr[2]
 130         movd            (@ptr[3]),@Xi[4]        # borrow @Xi[4]
 131          lea            `16*4`(@ptr[3]),@ptr[3]
 132         punpckldq       @Xi[3],@Xi[0]
 133          movd           `4*$j-16*4`(@ptr[0]),@Xi[1]
 134         punpckldq       @Xi[4],@Xi[2]
 135          movd           `4*$j-16*4`(@ptr[1]),$t3
 136         punpckldq       @Xi[2],@Xi[0]
 137          movd           `4*$j-16*4`(@ptr[2]),$t2
 138         pshufb          $tx,@Xi[0]
 139 ___
 140 $code.=<<___ if ($i<14);                        # just load input
 141          movd           `4*$j-16*4`(@ptr[3]),$t1
 142          punpckldq      $t2,@Xi[1]
 143         movdqa  $a,$t2
 144         paddd   $K,$e                           # e+=K_00_19
 145          punpckldq      $t1,$t3
 146         movdqa  $b,$t1
 147         movdqa  $b,$t0
 148         pslld   \$5,$t2
 149         pandn   $d,$t1
 150         pand    $c,$t0
 151          punpckldq      $t3,@Xi[1]
 152         movdqa  $a,$t3
 153
 154         movdqa  @Xi[0],`&Xi_off($i)`
 155         paddd   @Xi[0],$e                       # e+=X[i]
 156          movd           `4*$k-16*4`(@ptr[0]),@Xi[2]
 157         psrld   \$27,$t3
 158         pxor    $t1,$t0                         # Ch(b,c,d)
 159         movdqa  $b,$t1
 160
 161         por     $t3,$t2                         # rol(a,5)
 162          movd           `4*$k-16*4`(@ptr[1]),$t3
 163         pslld   \$30,$t1
 164         paddd   $t0,$e                          # e+=Ch(b,c,d)
 165
 166         psrld   \$2,$b
 167         paddd   $t2,$e                          # e+=rol(a,5)
 168          pshufb $tx,@Xi[1]
 169          movd           `4*$k-16*4`(@ptr[2]),$t2
 170         por     $t1,$b                          # b=rol(b,30)
 171 ___
 172 $code.=<<___ if ($i==14);                       # just load input
 173          movd           `4*$j-16*4`(@ptr[3]),$t1
 174          punpckldq      $t2,@Xi[1]
 175         movdqa  $a,$t2
 176         paddd   $K,$e                           # e+=K_00_19
 177          punpckldq      $t1,$t3
 178         movdqa  $b,$t1
 179         movdqa  $b,$t0
 180         pslld   \$5,$t2
 181          prefetcht0     63(@ptr[0])
 182         pandn   $d,$t1
 183         pand    $c,$t0
 184          punpckldq      $t3,@Xi[1]
 185         movdqa  $a,$t3
 186
 187         movdqa  @Xi[0],`&Xi_off($i)`
 188         paddd   @Xi[0],$e                       # e+=X[i]
 189         psrld   \$27,$t3
 190         pxor    $t1,$t0                         # Ch(b,c,d)
 191         movdqa  $b,$t1
 192          prefetcht0     63(@ptr[1])
 193
 194         por     $t3,$t2                         # rol(a,5)
 195         pslld   \$30,$t1
 196         paddd   $t0,$e                          # e+=Ch(b,c,d)
 197          prefetcht0     63(@ptr[2])
 198
 199         psrld   \$2,$b
 200         paddd   $t2,$e                          # e+=rol(a,5)
 201          pshufb $tx,@Xi[1]
 202          prefetcht0     63(@ptr[3])
 203         por     $t1,$b                          # b=rol(b,30)
 204 ___
 205 $code.=<<___ if ($i>=13 && $i<15);
 206         movdqa  `&Xi_off($j+2)`,@Xi[3]          # preload "X[2]"
 207 ___
 208 $code.=<<___ if ($i>=15);                       # apply Xupdate
 209         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 210         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 211
 212         movdqa  $a,$t2
 213          pxor   `&Xi_off($j+8)`,@Xi[1]
 214         paddd   $K,$e                           # e+=K_00_19
 215         movdqa  $b,$t1
 216         pslld   \$5,$t2
 217          pxor   @Xi[3],@Xi[1]
 218         movdqa  $b,$t0
 219         pandn   $d,$t1
 220          movdqa @Xi[1],$tx
 221         pand    $c,$t0
 222         movdqa  $a,$t3
 223          psrld  \$31,$tx
 224          paddd  @Xi[1],@Xi[1]
 225
 226         movdqa  @Xi[0],`&Xi_off($i)`
 227         paddd   @Xi[0],$e                       # e+=X[i]
 228         psrld   \$27,$t3
 229         pxor    $t1,$t0                         # Ch(b,c,d)
 230
 231         movdqa  $b,$t1
 232         por     $t3,$t2                         # rol(a,5)
 233         pslld   \$30,$t1
 234         paddd   $t0,$e                          # e+=Ch(b,c,d)
 235
 236         psrld   \$2,$b
 237         paddd   $t2,$e                          # e+=rol(a,5)
 238          por    $tx,@Xi[1]                      # rol   \$1,@Xi[1]
 239         por     $t1,$b                          # b=rol(b,30)
 240 ___
 241 push(@Xi,shift(@Xi));
 242 }
 243
 244 sub BODY_20_39 {
 245 my ($i,$a,$b,$c,$d,$e)=@_;
 246 my $j=$i+1;
 247
 248 $code.=<<___ if ($i<79);
 249         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 250         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 251
 252         movdqa  $a,$t2
 253         movdqa  $d,$t0
 254          pxor   `&Xi_off($j+8)`,@Xi[1]
 255         paddd   $K,$e                           # e+=K_20_39
 256         pslld   \$5,$t2
 257         pxor    $b,$t0
 258
 259         movdqa  $a,$t3
 260 ___
 261 $code.=<<___ if ($i<72);
 262         movdqa  @Xi[0],`&Xi_off($i)`
 263 ___
 264 $code.=<<___ if ($i<79);
 265         paddd   @Xi[0],$e                       # e+=X[i]
 266          pxor   @Xi[3],@Xi[1]
 267         psrld   \$27,$t3
 268         pxor    $c,$t0                          # Parity(b,c,d)
 269         movdqa  $b,$t1
 270
 271         pslld   \$30,$t1
 272          movdqa @Xi[1],$tx
 273         por     $t3,$t2                         # rol(a,5)
 274          psrld  \$31,$tx
 275         paddd   $t0,$e                          # e+=Parity(b,c,d)
 276          paddd  @Xi[1],@Xi[1]
 277
 278         psrld   \$2,$b
 279         paddd   $t2,$e                          # e+=rol(a,5)
 280          por    $tx,@Xi[1]                      # rol(@Xi[1],1)
 281         por     $t1,$b                          # b=rol(b,30)
 282 ___
 283 $code.=<<___ if ($i==79);
 284         movdqa  $a,$t2
 285         paddd   $K,$e                           # e+=K_20_39
 286         movdqa  $d,$t0
 287         pslld   \$5,$t2
 288         pxor    $b,$t0
 289
 290         movdqa  $a,$t3
 291         paddd   @Xi[0],$e                       # e+=X[i]
 292         psrld   \$27,$t3
 293         movdqa  $b,$t1
 294         pxor    $c,$t0                          # Parity(b,c,d)
 295
 296         pslld   \$30,$t1
 297         por     $t3,$t2                         # rol(a,5)
 298         paddd   $t0,$e                          # e+=Parity(b,c,d)
 299
 300         psrld   \$2,$b
 301         paddd   $t2,$e                          # e+=rol(a,5)
 302         por     $t1,$b                          # b=rol(b,30)
 303 ___
 304 push(@Xi,shift(@Xi));
 305 }
 306
 307 sub BODY_40_59 {
 308 my ($i,$a,$b,$c,$d,$e)=@_;
 309 my $j=$i+1;
 310
 311 $code.=<<___;
 312         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 313         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 314
 315         movdqa  $a,$t2
 316         movdqa  $d,$t1
 317          pxor   `&Xi_off($j+8)`,@Xi[1]
 318         pxor    @Xi[3],@Xi[1]
 319         paddd   $K,$e                           # e+=K_40_59
 320         pslld   \$5,$t2
 321         movdqa  $a,$t3
 322         pand    $c,$t1
 323
 324         movdqa  $d,$t0
 325          movdqa @Xi[1],$tx
 326         psrld   \$27,$t3
 327         paddd   $t1,$e
 328         pxor    $c,$t0
 329
 330         movdqa  @Xi[0],`&Xi_off($i)`
 331         paddd   @Xi[0],$e                       # e+=X[i]
 332         por     $t3,$t2                         # rol(a,5)
 333          psrld  \$31,$tx
 334         pand    $b,$t0
 335         movdqa  $b,$t1
 336
 337         pslld   \$30,$t1
 338          paddd  @Xi[1],@Xi[1]
 339         paddd   $t0,$e                          # e+=Maj(b,d,c)
 340
 341         psrld   \$2,$b
 342         paddd   $t2,$e                          # e+=rol(a,5)
 343          por    $tx,@Xi[1]                      # rol(@X[1],1)
 344         por     $t1,$b                          # b=rol(b,30)
 345 ___
 346 push(@Xi,shift(@Xi));
 347 }
 348
 349 $code.=<<___;
 350 .text
 351
 352 .extern OPENSSL_ia32cap_P
 353
 354 .globl  sha1_multi_block
 355 .type   sha1_multi_block,\@function,3
 356 .align  32
 357 sha1_multi_block:
 358         mov     OPENSSL_ia32cap_P+4(%rip),%rcx
 359         bt      \$61,%rcx                       # check SHA bit
 360         jc      _shaext_shortcut
 361 ___
 362 $code.=<<___ if ($avx);
 363         test    \$`1<<28`,%ecx
 364         jnz     _avx_shortcut
 365 ___
 366 $code.=<<___;
 367         mov     %rsp,%rax
 368         push    %rbx
 369         push    %rbp
 370 ___
 371 $code.=<<___ if ($win64);
 372         lea     -0xa8(%rsp),%rsp
 373         movaps  %xmm6,(%rsp)
 374         movaps  %xmm7,0x10(%rsp)
 375         movaps  %xmm8,0x20(%rsp)
 376         movaps  %xmm9,0x30(%rsp)
 377         movaps  %xmm10,-0x78(%rax)
 378         movaps  %xmm11,-0x68(%rax)
 379         movaps  %xmm12,-0x58(%rax)
 380         movaps  %xmm13,-0x48(%rax)
 381         movaps  %xmm14,-0x38(%rax)
 382         movaps  %xmm15,-0x28(%rax)
 383 ___
 384 $code.=<<___;
 385         sub     \$`$REG_SZ*18`,%rsp
 386         and     \$-256,%rsp
 387         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 388 .Lbody:
 389         lea     K_XX_XX(%rip),$Tbl
 390         lea     `$REG_SZ*16`(%rsp),%rbx
 391
 392 .Loop_grande:
 393         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
 394         xor     $num,$num
 395 ___
 396 for($i=0;$i<4;$i++) {
 397     $code.=<<___;
 398         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 399         mov     `16*$i+8`($inp),%ecx            # number of blocks
 400         cmp     $num,%ecx
 401         cmovg   %ecx,$num                       # find maximum
 402         test    %ecx,%ecx
 403         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 404         cmovle  $Tbl,@ptr[$i]                   # cancel input
 405 ___
 406 }
 407 $code.=<<___;
 408         test    $num,$num
 409         jz      .Ldone
 410
 411         movdqu  0x00($ctx),$A                   # load context
 412          lea    128(%rsp),%rax
 413         movdqu  0x20($ctx),$B
 414         movdqu  0x40($ctx),$C
 415         movdqu  0x60($ctx),$D
 416         movdqu  0x80($ctx),$E
 417         movdqa  0x60($Tbl),$tx                  # pbswap_mask
 418         movdqa  -0x20($Tbl),$K                  # K_00_19
 419         jmp     .Loop
 420
 421 .align  32
 422 .Loop:
 423 ___
 424 for($i=0;$i<20;$i++)    { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
 425 $code.="        movdqa  0x00($Tbl),$K\n";       # K_20_39
 426 for(;$i<40;$i++)        { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
 427 $code.="        movdqa  0x20($Tbl),$K\n";       # K_40_59
 428 for(;$i<60;$i++)        { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
 429 $code.="        movdqa  0x40($Tbl),$K\n";       # K_60_79
 430 for(;$i<80;$i++)        { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
 431 $code.=<<___;
 432         movdqa  (%rbx),@Xi[0]                   # pull counters
 433         mov     \$1,%ecx
 434         cmp     4*0(%rbx),%ecx                  # examinte counters
 435         pxor    $t2,$t2
 436         cmovge  $Tbl,@ptr[0]                    # cancel input
 437         cmp     4*1(%rbx),%ecx
 438         movdqa  @Xi[0],@Xi[1]
 439         cmovge  $Tbl,@ptr[1]
 440         cmp     4*2(%rbx),%ecx
 441         pcmpgtd $t2,@Xi[1]                      # mask value
 442         cmovge  $Tbl,@ptr[2]
 443         cmp     4*3(%rbx),%ecx
 444         paddd   @Xi[1],@Xi[0]                   # counters--
 445         cmovge  $Tbl,@ptr[3]
 446
 447         movdqu  0x00($ctx),$t0
 448         pand    @Xi[1],$A
 449         movdqu  0x20($ctx),$t1
 450         pand    @Xi[1],$B
 451         paddd   $t0,$A
 452         movdqu  0x40($ctx),$t2
 453         pand    @Xi[1],$C
 454         paddd   $t1,$B
 455         movdqu  0x60($ctx),$t3
 456         pand    @Xi[1],$D
 457         paddd   $t2,$C
 458         movdqu  0x80($ctx),$tx
 459         pand    @Xi[1],$E
 460         movdqu  $A,0x00($ctx)
 461         paddd   $t3,$D
 462         movdqu  $B,0x20($ctx)
 463         paddd   $tx,$E
 464         movdqu  $C,0x40($ctx)
 465         movdqu  $D,0x60($ctx)
 466         movdqu  $E,0x80($ctx)
 467
 468         movdqa  @Xi[0],(%rbx)                   # save counters
 469         movdqa  0x60($Tbl),$tx                  # pbswap_mask
 470         movdqa  -0x20($Tbl),$K                  # K_00_19
 471         dec     $num
 472         jnz     .Loop
 473
 474         mov     `$REG_SZ*17+8`(%rsp),$num
 475         lea     $REG_SZ($ctx),$ctx
 476         lea     `16*$REG_SZ/4`($inp),$inp
 477         dec     $num
 478         jnz     .Loop_grande
 479
 480 .Ldone:
 481         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
 482 ___
 483 $code.=<<___ if ($win64);
 484         movaps  -0xb8(%rax),%xmm6
 485         movaps  -0xa8(%rax),%xmm7
 486         movaps  -0x98(%rax),%xmm8
 487         movaps  -0x88(%rax),%xmm9
 488         movaps  -0x78(%rax),%xmm10
 489         movaps  -0x68(%rax),%xmm11
 490         movaps  -0x58(%rax),%xmm12
 491         movaps  -0x48(%rax),%xmm13
 492         movaps  -0x38(%rax),%xmm14
 493         movaps  -0x28(%rax),%xmm15
 494 ___
 495 $code.=<<___;
 496         mov     -16(%rax),%rbp
 497         mov     -8(%rax),%rbx
 498         lea     (%rax),%rsp
 499 .Lepilogue:
 500         ret
 501 .size   sha1_multi_block,.-sha1_multi_block
 502 ___
 503                                                 {{{
 504 my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10));
 505 my @MSG0=map("%xmm$_",(4..7));
 506 my @MSG1=map("%xmm$_",(11..14));
 507
 508 $code.=<<___;
 509 .type   sha1_multi_block_shaext,\@function,3
 510 .align  32
 511 sha1_multi_block_shaext:
 512 _shaext_shortcut:
 513         mov     %rsp,%rax
 514         push    %rbx
 515         push    %rbp
 516 ___
 517 $code.=<<___ if ($win64);
 518         lea     -0xa8(%rsp),%rsp
 519         movaps  %xmm6,(%rsp)
 520         movaps  %xmm7,0x10(%rsp)
 521         movaps  %xmm8,0x20(%rsp)
 522         movaps  %xmm9,0x30(%rsp)
 523         movaps  %xmm10,-0x78(%rax)
 524         movaps  %xmm11,-0x68(%rax)
 525         movaps  %xmm12,-0x58(%rax)
 526         movaps  %xmm13,-0x48(%rax)
 527         movaps  %xmm14,-0x38(%rax)
 528         movaps  %xmm15,-0x28(%rax)
 529 ___
 530 $code.=<<___;
 531         sub     \$`$REG_SZ*18`,%rsp
 532         shl     \$1,$num                        # we process pair at a time
 533         and     \$-256,%rsp
 534         lea     0x40($ctx),$ctx                 # size optimization
 535         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 536 .Lbody_shaext:
 537         lea     `$REG_SZ*16`(%rsp),%rbx
 538         movdqa  K_XX_XX+0x80(%rip),$BSWAP       # byte-n-word swap
 539
 540 .Loop_grande_shaext:
 541         mov     $num,`$REG_SZ*17+8`(%rsp)       # orignal $num
 542         xor     $num,$num
 543 ___
 544 for($i=0;$i<2;$i++) {
 545     $code.=<<___;
 546         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 547         mov     `16*$i+8`($inp),%ecx            # number of blocks
 548         cmp     $num,%ecx
 549         cmovg   %ecx,$num                       # find maximum
 550         test    %ecx,%ecx
 551         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 552         cmovle  %rsp,@ptr[$i]                   # cancel input
 553 ___
 554 }
 555 $code.=<<___;
 556         test    $num,$num
 557         jz      .Ldone_shaext
 558
 559         movq            0x00-0x40($ctx),$ABCD0  # a1.a0
 560         movq            0x20-0x40($ctx),@MSG0[0]# b1.b0
 561         movq            0x40-0x40($ctx),@MSG0[1]# c1.c0
 562         movq            0x60-0x40($ctx),@MSG0[2]# d1.d0
 563         movq            0x80-0x40($ctx),@MSG0[3]# e1.e0
 564
 565         punpckldq       @MSG0[0],$ABCD0         # b1.a1.b0.a0
 566         punpckldq       @MSG0[2],@MSG0[1]       # d1.c1.d0.c0
 567
 568         movdqa          $ABCD0,$ABCD1
 569         punpcklqdq      @MSG0[1],$ABCD0         # d0.c0.b0.a0
 570         punpckhqdq      @MSG0[1],$ABCD1         # d1.c1.b1.a1
 571
 572         pshufd          \$0b00111111,@MSG0[3],$E0
 573         pshufd          \$0b01111111,@MSG0[3],$E1
 574         pshufd          \$0b00011011,$ABCD0,$ABCD0
 575         pshufd          \$0b00011011,$ABCD1,$ABCD1
 576         jmp             .Loop_shaext
 577
 578 .align  32
 579 .Loop_shaext:
 580         movdqu          0x00(@ptr[0]),@MSG0[0]
 581          movdqu         0x00(@ptr[1]),@MSG1[0]
 582         movdqu          0x10(@ptr[0]),@MSG0[1]
 583          movdqu         0x10(@ptr[1]),@MSG1[1]
 584         movdqu          0x20(@ptr[0]),@MSG0[2]
 585         pshufb          $BSWAP,@MSG0[0]
 586          movdqu         0x20(@ptr[1]),@MSG1[2]
 587          pshufb         $BSWAP,@MSG1[0]
 588         movdqu          0x30(@ptr[0]),@MSG0[3]
 589         lea             0x40(@ptr[0]),@ptr[0]
 590         pshufb          $BSWAP,@MSG0[1]
 591          movdqu         0x30(@ptr[1]),@MSG1[3]
 592          lea            0x40(@ptr[1]),@ptr[1]
 593          pshufb         $BSWAP,@MSG1[1]
 594
 595         movdqa          $E0,0x50(%rsp)          # offload
 596         paddd           @MSG0[0],$E0
 597          movdqa         $E1,0x70(%rsp)
 598          paddd          @MSG1[0],$E1
 599         movdqa          $ABCD0,0x40(%rsp)       # offload
 600         movdqa          $ABCD0,$E0_
 601          movdqa         $ABCD1,0x60(%rsp)
 602          movdqa         $ABCD1,$E1_
 603         sha1rnds4       \$0,$E0,$ABCD0          # 0-3
 604         sha1nexte       @MSG0[1],$E0_
 605          sha1rnds4      \$0,$E1,$ABCD1          # 0-3
 606          sha1nexte      @MSG1[1],$E1_
 607         pshufb          $BSWAP,@MSG0[2]
 608         prefetcht0      127(@ptr[0])
 609         sha1msg1        @MSG0[1],@MSG0[0]
 610          pshufb         $BSWAP,@MSG1[2]
 611          prefetcht0     127(@ptr[1])
 612          sha1msg1       @MSG1[1],@MSG1[0]
 613
 614         pshufb          $BSWAP,@MSG0[3]
 615         movdqa          $ABCD0,$E0
 616          pshufb         $BSWAP,@MSG1[3]
 617          movdqa         $ABCD1,$E1
 618         sha1rnds4       \$0,$E0_,$ABCD0         # 4-7
 619         sha1nexte       @MSG0[2],$E0
 620          sha1rnds4      \$0,$E1_,$ABCD1         # 4-7
 621          sha1nexte      @MSG1[2],$E1
 622         pxor            @MSG0[2],@MSG0[0]
 623         sha1msg1        @MSG0[2],@MSG0[1]
 624          pxor           @MSG1[2],@MSG1[0]
 625          sha1msg1       @MSG1[2],@MSG1[1]
 626 ___
 627 for($i=2;$i<20-4;$i++) {
 628 $code.=<<___;
 629         movdqa          $ABCD0,$E0_
 630          movdqa         $ABCD1,$E1_
 631         sha1rnds4       \$`int($i/5)`,$E0,$ABCD0        # 8-11
 632         sha1nexte       @MSG0[3],$E0_
 633          sha1rnds4      \$`int($i/5)`,$E1,$ABCD1        # 8-11
 634          sha1nexte      @MSG1[3],$E1_
 635         sha1msg2        @MSG0[3],@MSG0[0]
 636          sha1msg2       @MSG1[3],@MSG1[0]
 637         pxor            @MSG0[3],@MSG0[1]
 638         sha1msg1        @MSG0[3],@MSG0[2]
 639          pxor           @MSG1[3],@MSG1[1]
 640          sha1msg1       @MSG1[3],@MSG1[2]
 641 ___
 642         ($E0,$E0_)=($E0_,$E0);          ($E1,$E1_)=($E1_,$E1);
 643         push(@MSG0,shift(@MSG0));       push(@MSG1,shift(@MSG1));
 644 }
 645 $code.=<<___;
 646         movdqa          $ABCD0,$E0_
 647          movdqa         $ABCD1,$E1_
 648         sha1rnds4       \$3,$E0,$ABCD0          # 64-67
 649         sha1nexte       @MSG0[3],$E0_
 650          sha1rnds4      \$3,$E1,$ABCD1          # 64-67
 651          sha1nexte      @MSG1[3],$E1_
 652         sha1msg2        @MSG0[3],@MSG0[0]
 653          sha1msg2       @MSG1[3],@MSG1[0]
 654         pxor            @MSG0[3],@MSG0[1]
 655          pxor           @MSG1[3],@MSG1[1]
 656
 657         mov             \$1,%ecx
 658         pxor            @MSG0[2],@MSG0[2]       # zero
 659         cmp             4*0(%rbx),%ecx          # examine counters
 660         cmovge          %rsp,@ptr[0]            # cancel input
 661
 662         movdqa          $ABCD0,$E0
 663          movdqa         $ABCD1,$E1
 664         sha1rnds4       \$3,$E0_,$ABCD0         # 68-71
 665         sha1nexte       @MSG0[0],$E0
 666          sha1rnds4      \$3,$E1_,$ABCD1         # 68-71
 667          sha1nexte      @MSG1[0],$E1
 668         sha1msg2        @MSG0[0],@MSG0[1]
 669          sha1msg2       @MSG1[0],@MSG1[1]
 670
 671         cmp             4*1(%rbx),%ecx
 672         cmovge          %rsp,@ptr[1]
 673         movq            (%rbx),@MSG0[0]         # pull counters
 674
 675         movdqa          $ABCD0,$E0_
 676          movdqa         $ABCD1,$E1_
 677         sha1rnds4       \$3,$E0,$ABCD0          # 72-75
 678         sha1nexte       @MSG0[1],$E0_
 679          sha1rnds4      \$3,$E1,$ABCD1          # 72-75
 680          sha1nexte      @MSG1[1],$E1_
 681
 682         pshufd          \$0x00,@MSG0[0],@MSG1[2]
 683         pshufd          \$0x55,@MSG0[0],@MSG1[3]
 684         movdqa          @MSG0[0],@MSG0[1]
 685         pcmpgtd         @MSG0[2],@MSG1[2]
 686         pcmpgtd         @MSG0[2],@MSG1[3]
 687
 688         movdqa          $ABCD0,$E0
 689          movdqa         $ABCD1,$E1
 690         sha1rnds4       \$3,$E0_,$ABCD0         # 76-79
 691         sha1nexte       $MSG0[2],$E0
 692          sha1rnds4      \$3,$E1_,$ABCD1         # 76-79
 693          sha1nexte      $MSG0[2],$E1
 694
 695         pcmpgtd         @MSG0[2],@MSG0[1]       # counter mask
 696         pand            @MSG1[2],$ABCD0
 697         pand            @MSG1[2],$E0
 698          pand           @MSG1[3],$ABCD1
 699          pand           @MSG1[3],$E1
 700         paddd           @MSG0[1],@MSG0[0]       # counters--
 701
 702         paddd           0x40(%rsp),$ABCD0
 703         paddd           0x50(%rsp),$E0
 704          paddd          0x60(%rsp),$ABCD1
 705          paddd          0x70(%rsp),$E1
 706
 707         movq            @MSG0[0],(%rbx)         # save counters
 708         dec             $num
 709         jnz             .Loop_shaext
 710
 711         mov             `$REG_SZ*17+8`(%rsp),$num
 712
 713         pshufd          \$0b00011011,$ABCD0,$ABCD0
 714         pshufd          \$0b00011011,$ABCD1,$ABCD1
 715
 716         movdqa          $ABCD0,@MSG0[0]
 717         punpckldq       $ABCD1,$ABCD0           # b1.b0.a1.a0
 718         punpckhdq       $ABCD1,@MSG0[0]         # d1.d0.c1.c0
 719         punpckhdq       $E1,$E0                 # e1.e0.xx.xx
 720         movq            $ABCD0,0x00-0x40($ctx)  # a1.a0
 721         psrldq          \$8,$ABCD0
 722         movq            @MSG0[0],0x40-0x40($ctx)# c1.c0
 723         psrldq          \$8,@MSG0[0]
 724         movq            $ABCD0,0x20-0x40($ctx)  # b1.b0
 725         psrldq          \$8,$E0
 726         movq            @MSG0[0],0x60-0x40($ctx)# d1.d0
 727         movq            $E0,0x80-0x40($ctx)     # e1.e0
 728
 729         lea     `$REG_SZ/2`($ctx),$ctx
 730         lea     `16*2`($inp),$inp
 731         dec     $num
 732         jnz     .Loop_grande_shaext
 733
 734 .Ldone_shaext:
 735         #mov    `$REG_SZ*17`(%rsp),%rax         # original %rsp
 736 ___
 737 $code.=<<___ if ($win64);
 738         movaps  -0xb8(%rax),%xmm6
 739         movaps  -0xa8(%rax),%xmm7
 740         movaps  -0x98(%rax),%xmm8
 741         movaps  -0x88(%rax),%xmm9
 742         movaps  -0x78(%rax),%xmm10
 743         movaps  -0x68(%rax),%xmm11
 744         movaps  -0x58(%rax),%xmm12
 745         movaps  -0x48(%rax),%xmm13
 746         movaps  -0x38(%rax),%xmm14
 747         movaps  -0x28(%rax),%xmm15
 748 ___
 749 $code.=<<___;
 750         mov     -16(%rax),%rbp
 751         mov     -8(%rax),%rbx
 752         lea     (%rax),%rsp
 753 .Lepilogue_shaext:
 754         ret
 755 .size   sha1_multi_block_shaext,.-sha1_multi_block_shaext
 756 ___
 757                                                 }}}
 758
 759                                                 if ($avx) {{{
 760 sub BODY_00_19_avx {
 761 my ($i,$a,$b,$c,$d,$e)=@_;
 762 my $j=$i+1;
 763 my $k=$i+2;
 764 my $vpack = $REG_SZ==16 ? "vpunpckldq" : "vinserti128";
 765 my $ptr_n = $REG_SZ==16 ? @ptr[1] : @ptr[4];
 766
 767 $code.=<<___ if ($i==0 && $REG_SZ==16);
 768         vmovd           (@ptr[0]),@Xi[0]
 769          lea            `16*4`(@ptr[0]),@ptr[0]
 770         vmovd           (@ptr[1]),@Xi[2]        # borrow Xi[2]
 771          lea            `16*4`(@ptr[1]),@ptr[1]
 772         vpinsrd         \$1,(@ptr[2]),@Xi[0],@Xi[0]
 773          lea            `16*4`(@ptr[2]),@ptr[2]
 774         vpinsrd         \$1,(@ptr[3]),@Xi[2],@Xi[2]
 775          lea            `16*4`(@ptr[3]),@ptr[3]
 776          vmovd          `4*$j-16*4`(@ptr[0]),@Xi[1]
 777         vpunpckldq      @Xi[2],@Xi[0],@Xi[0]
 778          vmovd          `4*$j-16*4`($ptr_n),$t3
 779         vpshufb         $tx,@Xi[0],@Xi[0]
 780 ___
 781 $code.=<<___ if ($i<15 && $REG_SZ==16);         # just load input
 782          vpinsrd        \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
 783          vpinsrd        \$1,`4*$j-16*4`(@ptr[3]),$t3,$t3
 784 ___
 785 $code.=<<___ if ($i==0 && $REG_SZ==32);
 786         vmovd           (@ptr[0]),@Xi[0]
 787          lea            `16*4`(@ptr[0]),@ptr[0]
 788         vmovd           (@ptr[4]),@Xi[2]        # borrow Xi[2]
 789          lea            `16*4`(@ptr[4]),@ptr[4]
 790         vmovd           (@ptr[1]),$t2
 791          lea            `16*4`(@ptr[1]),@ptr[1]
 792         vmovd           (@ptr[5]),$t1
 793          lea            `16*4`(@ptr[5]),@ptr[5]
 794         vpinsrd         \$1,(@ptr[2]),@Xi[0],@Xi[0]
 795          lea            `16*4`(@ptr[2]),@ptr[2]
 796         vpinsrd         \$1,(@ptr[6]),@Xi[2],@Xi[2]
 797          lea            `16*4`(@ptr[6]),@ptr[6]
 798         vpinsrd         \$1,(@ptr[3]),$t2,$t2
 799          lea            `16*4`(@ptr[3]),@ptr[3]
 800         vpunpckldq      $t2,@Xi[0],@Xi[0]
 801         vpinsrd         \$1,(@ptr[7]),$t1,$t1
 802          lea            `16*4`(@ptr[7]),@ptr[7]
 803         vpunpckldq      $t1,@Xi[2],@Xi[2]
 804          vmovd          `4*$j-16*4`(@ptr[0]),@Xi[1]
 805         vinserti128     @Xi[2],@Xi[0],@Xi[0]
 806          vmovd          `4*$j-16*4`($ptr_n),$t3
 807         vpshufb         $tx,@Xi[0],@Xi[0]
 808 ___
 809 $code.=<<___ if ($i<15 && $REG_SZ==32);         # just load input
 810          vmovd          `4*$j-16*4`(@ptr[1]),$t2
 811          vmovd          `4*$j-16*4`(@ptr[5]),$t1
 812          vpinsrd        \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
 813          vpinsrd        \$1,`4*$j-16*4`(@ptr[6]),$t3,$t3
 814          vpinsrd        \$1,`4*$j-16*4`(@ptr[3]),$t2,$t2
 815          vpunpckldq     $t2,@Xi[1],@Xi[1]
 816          vpinsrd        \$1,`4*$j-16*4`(@ptr[7]),$t1,$t1
 817          vpunpckldq     $t1,$t3,$t3
 818 ___
 819 $code.=<<___ if ($i<14);
 820         vpaddd  $K,$e,$e                        # e+=K_00_19
 821         vpslld  \$5,$a,$t2
 822         vpandn  $d,$b,$t1
 823         vpand   $c,$b,$t0
 824
 825         vmovdqa @Xi[0],`&Xi_off($i)`
 826         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 827          $vpack         $t3,@Xi[1],@Xi[1]
 828         vpsrld  \$27,$a,$t3
 829         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 830          vmovd          `4*$k-16*4`(@ptr[0]),@Xi[2]
 831
 832         vpslld  \$30,$b,$t1
 833         vpor    $t3,$t2,$t2                     # rol(a,5)
 834          vmovd          `4*$k-16*4`($ptr_n),$t3
 835         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 836
 837         vpsrld  \$2,$b,$b
 838         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 839          vpshufb        $tx,@Xi[1],@Xi[1]
 840         vpor    $t1,$b,$b                       # b=rol(b,30)
 841 ___
 842 $code.=<<___ if ($i==14);
 843         vpaddd  $K,$e,$e                        # e+=K_00_19
 844          prefetcht0     63(@ptr[0])
 845         vpslld  \$5,$a,$t2
 846         vpandn  $d,$b,$t1
 847         vpand   $c,$b,$t0
 848
 849         vmovdqa @Xi[0],`&Xi_off($i)`
 850         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 851          $vpack         $t3,@Xi[1],@Xi[1]
 852         vpsrld  \$27,$a,$t3
 853          prefetcht0     63(@ptr[1])
 854         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 855
 856         vpslld  \$30,$b,$t1
 857         vpor    $t3,$t2,$t2                     # rol(a,5)
 858          prefetcht0     63(@ptr[2])
 859         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 860
 861         vpsrld  \$2,$b,$b
 862         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 863          prefetcht0     63(@ptr[3])
 864          vpshufb        $tx,@Xi[1],@Xi[1]
 865         vpor    $t1,$b,$b                       # b=rol(b,30)
 866 ___
 867 $code.=<<___ if ($i>=13 && $i<15);
 868         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # preload "X[2]"
 869 ___
 870 $code.=<<___ if ($i>=15);                       # apply Xupdate
 871         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 872         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 873
 874         vpaddd  $K,$e,$e                        # e+=K_00_19
 875         vpslld  \$5,$a,$t2
 876         vpandn  $d,$b,$t1
 877          `"prefetcht0   63(@ptr[4])"            if ($i==15 && $REG_SZ==32)`
 878         vpand   $c,$b,$t0
 879
 880         vmovdqa @Xi[0],`&Xi_off($i)`
 881         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 882          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 883         vpsrld  \$27,$a,$t3
 884         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 885          vpxor  @Xi[3],@Xi[1],@Xi[1]
 886          `"prefetcht0   63(@ptr[5])"            if ($i==15 && $REG_SZ==32)`
 887
 888         vpslld  \$30,$b,$t1
 889         vpor    $t3,$t2,$t2                     # rol(a,5)
 890         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 891          `"prefetcht0   63(@ptr[6])"            if ($i==15 && $REG_SZ==32)`
 892          vpsrld \$31,@Xi[1],$tx
 893          vpaddd @Xi[1],@Xi[1],@Xi[1]
 894
 895         vpsrld  \$2,$b,$b
 896          `"prefetcht0   63(@ptr[7])"            if ($i==15 && $REG_SZ==32)`
 897         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 898          vpor   $tx,@Xi[1],@Xi[1]               # rol   \$1,@Xi[1]
 899         vpor    $t1,$b,$b                       # b=rol(b,30)
 900 ___
 901 push(@Xi,shift(@Xi));
 902 }
 903
 904 sub BODY_20_39_avx {
 905 my ($i,$a,$b,$c,$d,$e)=@_;
 906 my $j=$i+1;
 907
 908 $code.=<<___ if ($i<79);
 909         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 910         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 911
 912         vpslld  \$5,$a,$t2
 913         vpaddd  $K,$e,$e                        # e+=K_20_39
 914         vpxor   $b,$d,$t0
 915 ___
 916 $code.=<<___ if ($i<72);
 917         vmovdqa @Xi[0],`&Xi_off($i)`
 918 ___
 919 $code.=<<___ if ($i<79);
 920         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 921          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 922         vpsrld  \$27,$a,$t3
 923         vpxor   $c,$t0,$t0                      # Parity(b,c,d)
 924          vpxor  @Xi[3],@Xi[1],@Xi[1]
 925
 926         vpslld  \$30,$b,$t1
 927         vpor    $t3,$t2,$t2                     # rol(a,5)
 928         vpaddd  $t0,$e,$e                       # e+=Parity(b,c,d)
 929          vpsrld \$31,@Xi[1],$tx
 930          vpaddd @Xi[1],@Xi[1],@Xi[1]
 931
 932         vpsrld  \$2,$b,$b
 933         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 934          vpor   $tx,@Xi[1],@Xi[1]               # rol(@Xi[1],1)
 935         vpor    $t1,$b,$b                       # b=rol(b,30)
 936 ___
 937 $code.=<<___ if ($i==79);
 938         vpslld  \$5,$a,$t2
 939         vpaddd  $K,$e,$e                        # e+=K_20_39
 940         vpxor   $b,$d,$t0
 941
 942         vpsrld  \$27,$a,$t3
 943         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 944         vpxor   $c,$t0,$t0                      # Parity(b,c,d)
 945
 946         vpslld  \$30,$b,$t1
 947         vpor    $t3,$t2,$t2                     # rol(a,5)
 948         vpaddd  $t0,$e,$e                       # e+=Parity(b,c,d)
 949
 950         vpsrld  \$2,$b,$b
 951         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 952         vpor    $t1,$b,$b                       # b=rol(b,30)
 953 ___
 954 push(@Xi,shift(@Xi));
 955 }
 956
 957 sub BODY_40_59_avx {
 958 my ($i,$a,$b,$c,$d,$e)=@_;
 959 my $j=$i+1;
 960
 961 $code.=<<___;
 962         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 963         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 964
 965         vpaddd  $K,$e,$e                        # e+=K_40_59
 966         vpslld  \$5,$a,$t2
 967         vpand   $c,$d,$t1
 968          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 969
 970         vpaddd  $t1,$e,$e
 971         vpsrld  \$27,$a,$t3
 972         vpxor   $c,$d,$t0
 973          vpxor  @Xi[3],@Xi[1],@Xi[1]
 974
 975         vmovdqu @Xi[0],`&Xi_off($i)`
 976         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 977         vpor    $t3,$t2,$t2                     # rol(a,5)
 978          vpsrld \$31,@Xi[1],$tx
 979         vpand   $b,$t0,$t0
 980          vpaddd @Xi[1],@Xi[1],@Xi[1]
 981
 982         vpslld  \$30,$b,$t1
 983         vpaddd  $t0,$e,$e                       # e+=Maj(b,d,c)
 984
 985         vpsrld  \$2,$b,$b
 986         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 987          vpor   $tx,@Xi[1],@Xi[1]               # rol(@X[1],1)
 988         vpor    $t1,$b,$b                       # b=rol(b,30)
 989 ___
 990 push(@Xi,shift(@Xi));
 991 }
 992
 993 $code.=<<___;
 994 .type   sha1_multi_block_avx,\@function,3
 995 .align  32
 996 sha1_multi_block_avx:
 997 _avx_shortcut:
 998 ___
 999 $code.=<<___ if ($avx>1);
1000         shr     \$32,%rcx
1001         cmp     \$2,$num
1002         jb      .Lavx
1003         test    \$`1<<5`,%ecx
1004         jnz     _avx2_shortcut
1005         jmp     .Lavx
1006 .align  32
1007 .Lavx:
1008 ___
1009 $code.=<<___;
1010         mov     %rsp,%rax
1011         push    %rbx
1012         push    %rbp
1013 ___
1014 $code.=<<___ if ($win64);
1015         lea     -0xa8(%rsp),%rsp
1016         movaps  %xmm6,(%rsp)
1017         movaps  %xmm7,0x10(%rsp)
1018         movaps  %xmm8,0x20(%rsp)
1019         movaps  %xmm9,0x30(%rsp)
1020         movaps  %xmm10,-0x78(%rax)
1021         movaps  %xmm11,-0x68(%rax)
1022         movaps  %xmm12,-0x58(%rax)
1023         movaps  %xmm13,-0x48(%rax)
1024         movaps  %xmm14,-0x38(%rax)
1025         movaps  %xmm15,-0x28(%rax)
1026 ___
1027 $code.=<<___;
1028         sub     \$`$REG_SZ*18`, %rsp
1029         and     \$-256,%rsp
1030         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1031 .Lbody_avx:
1032         lea     K_XX_XX(%rip),$Tbl
1033         lea     `$REG_SZ*16`(%rsp),%rbx
1034
1035         vzeroupper
1036 .Loop_grande_avx:
1037         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1038         xor     $num,$num
1039 ___
1040 for($i=0;$i<4;$i++) {
1041     $code.=<<___;
1042         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1043         mov     `16*$i+8`($inp),%ecx            # number of blocks
1044         cmp     $num,%ecx
1045         cmovg   %ecx,$num                       # find maximum
1046         test    %ecx,%ecx
1047         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1048         cmovle  $Tbl,@ptr[$i]                   # cancel input
1049 ___
1050 }
1051 $code.=<<___;
1052         test    $num,$num
1053         jz      .Ldone_avx
1054
1055         vmovdqu 0x00($ctx),$A                   # load context
1056          lea    128(%rsp),%rax
1057         vmovdqu 0x20($ctx),$B
1058         vmovdqu 0x40($ctx),$C
1059         vmovdqu 0x60($ctx),$D
1060         vmovdqu 0x80($ctx),$E
1061         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1062         jmp     .Loop_avx
1063
1064 .align  32
1065 .Loop_avx:
1066 ___
1067 $code.="        vmovdqa -0x20($Tbl),$K\n";      # K_00_19
1068 for($i=0;$i<20;$i++)    { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1069 $code.="        vmovdqa 0x00($Tbl),$K\n";       # K_20_39
1070 for(;$i<40;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1071 $code.="        vmovdqa 0x20($Tbl),$K\n";       # K_40_59
1072 for(;$i<60;$i++)        { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1073 $code.="        vmovdqa 0x40($Tbl),$K\n";       # K_60_79
1074 for(;$i<80;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1075 $code.=<<___;
1076         mov     \$1,%ecx
1077 ___
1078 for($i=0;$i<4;$i++) {
1079     $code.=<<___;
1080         cmp     `4*$i`(%rbx),%ecx               # examine counters
1081         cmovge  $Tbl,@ptr[$i]                   # cancel input
1082 ___
1083 }
1084 $code.=<<___;
1085         vmovdqu (%rbx),$t0                      # pull counters
1086         vpxor   $t2,$t2,$t2
1087         vmovdqa $t0,$t1
1088         vpcmpgtd $t2,$t1,$t1                    # mask value
1089         vpaddd  $t1,$t0,$t0                     # counters--
1090
1091         vpand   $t1,$A,$A
1092         vpand   $t1,$B,$B
1093         vpaddd  0x00($ctx),$A,$A
1094         vpand   $t1,$C,$C
1095         vpaddd  0x20($ctx),$B,$B
1096         vpand   $t1,$D,$D
1097         vpaddd  0x40($ctx),$C,$C
1098         vpand   $t1,$E,$E
1099         vpaddd  0x60($ctx),$D,$D
1100         vpaddd  0x80($ctx),$E,$E
1101         vmovdqu $A,0x00($ctx)
1102         vmovdqu $B,0x20($ctx)
1103         vmovdqu $C,0x40($ctx)
1104         vmovdqu $D,0x60($ctx)
1105         vmovdqu $E,0x80($ctx)
1106
1107         vmovdqu $t0,(%rbx)                      # save counters
1108         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1109         dec     $num
1110         jnz     .Loop_avx
1111
1112         mov     `$REG_SZ*17+8`(%rsp),$num
1113         lea     $REG_SZ($ctx),$ctx
1114         lea     `16*$REG_SZ/4`($inp),$inp
1115         dec     $num
1116         jnz     .Loop_grande_avx
1117
1118 .Ldone_avx:
1119         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
1120         vzeroupper
1121 ___
1122 $code.=<<___ if ($win64);
1123         movaps  -0xb8(%rax),%xmm6
1124         movaps  -0xa8(%rax),%xmm7
1125         movaps  -0x98(%rax),%xmm8
1126         movaps  -0x88(%rax),%xmm9
1127         movaps  -0x78(%rax),%xmm10
1128         movaps  -0x68(%rax),%xmm11
1129         movaps  -0x58(%rax),%xmm12
1130         movaps  -0x48(%rax),%xmm13
1131         movaps  -0x38(%rax),%xmm14
1132         movaps  -0x28(%rax),%xmm15
1133 ___
1134 $code.=<<___;
1135         mov     -16(%rax),%rbp
1136         mov     -8(%rax),%rbx
1137         lea     (%rax),%rsp
1138 .Lepilogue_avx:
1139         ret
1140 .size   sha1_multi_block_avx,.-sha1_multi_block_avx
1141 ___
1142
1143                                                 if ($avx>1) {
1144 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1145
1146 $REG_SZ=32;
1147
1148 @ptr=map("%r$_",(12..15,8..11));
1149
1150 @V=($A,$B,$C,$D,$E)=map("%ymm$_",(0..4));
1151 ($t0,$t1,$t2,$t3,$tx)=map("%ymm$_",(5..9));
1152 @Xi=map("%ymm$_",(10..14));
1153 $K="%ymm15";
1154
1155 $code.=<<___;
1156 .type   sha1_multi_block_avx2,\@function,3
1157 .align  32
1158 sha1_multi_block_avx2:
1159 _avx2_shortcut:
1160         mov     %rsp,%rax
1161         push    %rbx
1162         push    %rbp
1163         push    %r12
1164         push    %r13
1165         push    %r14
1166         push    %r15
1167 ___
1168 $code.=<<___ if ($win64);
1169         lea     -0xa8(%rsp),%rsp
1170         movaps  %xmm6,(%rsp)
1171         movaps  %xmm7,0x10(%rsp)
1172         movaps  %xmm8,0x20(%rsp)
1173         movaps  %xmm9,0x30(%rsp)
1174         movaps  %xmm10,0x40(%rsp)
1175         movaps  %xmm11,0x50(%rsp)
1176         movaps  %xmm12,-0x78(%rax)
1177         movaps  %xmm13,-0x68(%rax)
1178         movaps  %xmm14,-0x58(%rax)
1179         movaps  %xmm15,-0x48(%rax)
1180 ___
1181 $code.=<<___;
1182         sub     \$`$REG_SZ*18`, %rsp
1183         and     \$-256,%rsp
1184         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1185 .Lbody_avx2:
1186         lea     K_XX_XX(%rip),$Tbl
1187         shr     \$1,$num
1188
1189         vzeroupper
1190 .Loop_grande_avx2:
1191         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1192         xor     $num,$num
1193         lea     `$REG_SZ*16`(%rsp),%rbx
1194 ___
1195 for($i=0;$i<8;$i++) {
1196     $code.=<<___;
1197         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1198         mov     `16*$i+8`($inp),%ecx            # number of blocks
1199         cmp     $num,%ecx
1200         cmovg   %ecx,$num                       # find maximum
1201         test    %ecx,%ecx
1202         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1203         cmovle  $Tbl,@ptr[$i]                   # cancel input
1204 ___
1205 }
1206 $code.=<<___;
1207         vmovdqu 0x00($ctx),$A                   # load context
1208          lea    128(%rsp),%rax
1209         vmovdqu 0x20($ctx),$B
1210          lea    256+128(%rsp),%rbx
1211         vmovdqu 0x40($ctx),$C
1212         vmovdqu 0x60($ctx),$D
1213         vmovdqu 0x80($ctx),$E
1214         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1215         jmp     .Loop_avx2
1216
1217 .align  32
1218 .Loop_avx2:
1219 ___
1220 $code.="        vmovdqa -0x20($Tbl),$K\n";      # K_00_19
1221 for($i=0;$i<20;$i++)    { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1222 $code.="        vmovdqa 0x00($Tbl),$K\n";       # K_20_39
1223 for(;$i<40;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1224 $code.="        vmovdqa 0x20($Tbl),$K\n";       # K_40_59
1225 for(;$i<60;$i++)        { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1226 $code.="        vmovdqa 0x40($Tbl),$K\n";       # K_60_79
1227 for(;$i<80;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1228 $code.=<<___;
1229         mov     \$1,%ecx
1230         lea     `$REG_SZ*16`(%rsp),%rbx
1231 ___
1232 for($i=0;$i<8;$i++) {
1233     $code.=<<___;
1234         cmp     `4*$i`(%rbx),%ecx               # examine counters
1235         cmovge  $Tbl,@ptr[$i]                   # cancel input
1236 ___
1237 }
1238 $code.=<<___;
1239         vmovdqu (%rbx),$t0              # pull counters
1240         vpxor   $t2,$t2,$t2
1241         vmovdqa $t0,$t1
1242         vpcmpgtd $t2,$t1,$t1                    # mask value
1243         vpaddd  $t1,$t0,$t0                     # counters--
1244
1245         vpand   $t1,$A,$A
1246         vpand   $t1,$B,$B
1247         vpaddd  0x00($ctx),$A,$A
1248         vpand   $t1,$C,$C
1249         vpaddd  0x20($ctx),$B,$B
1250         vpand   $t1,$D,$D
1251         vpaddd  0x40($ctx),$C,$C
1252         vpand   $t1,$E,$E
1253         vpaddd  0x60($ctx),$D,$D
1254         vpaddd  0x80($ctx),$E,$E
1255         vmovdqu $A,0x00($ctx)
1256         vmovdqu $B,0x20($ctx)
1257         vmovdqu $C,0x40($ctx)
1258         vmovdqu $D,0x60($ctx)
1259         vmovdqu $E,0x80($ctx)
1260
1261         vmovdqu $t0,(%rbx)                      # save counters
1262         lea     256+128(%rsp),%rbx
1263         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1264         dec     $num
1265         jnz     .Loop_avx2
1266
1267         #mov    `$REG_SZ*17+8`(%rsp),$num
1268         #lea    $REG_SZ($ctx),$ctx
1269         #lea    `16*$REG_SZ/4`($inp),$inp
1270         #dec    $num
1271         #jnz    .Loop_grande_avx2
1272
1273 .Ldone_avx2:
1274         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
1275         vzeroupper
1276 ___
1277 $code.=<<___ if ($win64);
1278         movaps  -0xd8(%rax),%xmm6
1279         movaps  -0xc8(%rax),%xmm7
1280         movaps  -0xb8(%rax),%xmm8
1281         movaps  -0xa8(%rax),%xmm9
1282         movaps  -0x98(%rax),%xmm10
1283         movaps  -0x88(%rax),%xmm11
1284         movaps  -0x78(%rax),%xmm12
1285         movaps  -0x68(%rax),%xmm13
1286         movaps  -0x58(%rax),%xmm14
1287         movaps  -0x48(%rax),%xmm15
1288 ___
1289 $code.=<<___;
1290         mov     -48(%rax),%r15
1291         mov     -40(%rax),%r14
1292         mov     -32(%rax),%r13
1293         mov     -24(%rax),%r12
1294         mov     -16(%rax),%rbp
1295         mov     -8(%rax),%rbx
1296         lea     (%rax),%rsp
1297 .Lepilogue_avx2:
1298         ret
1299 .size   sha1_multi_block_avx2,.-sha1_multi_block_avx2
1300 ___
1301                                                 }       }}}
1302 $code.=<<___;
1303
1304 .align  256
1305         .long   0x5a827999,0x5a827999,0x5a827999,0x5a827999     # K_00_19
1306         .long   0x5a827999,0x5a827999,0x5a827999,0x5a827999     # K_00_19
1307 K_XX_XX:
1308         .long   0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1     # K_20_39
1309         .long   0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1     # K_20_39
1310         .long   0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc     # K_40_59
1311         .long   0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc     # K_40_59
1312         .long   0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6     # K_60_79
1313         .long   0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6     # K_60_79
1314         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1315         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1316         .byte   0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0
1317         .asciz  "SHA1 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
1318 ___
1319
1320 if ($win64) {
1321 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1322 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1323 $rec="%rcx";
1324 $frame="%rdx";
1325 $context="%r8";
1326 $disp="%r9";
1327
1328 $code.=<<___;
1329 .extern __imp_RtlVirtualUnwind
1330 .type   se_handler,\@abi-omnipotent
1331 .align  16
1332 se_handler:
1333         push    %rsi
1334         push    %rdi
1335         push    %rbx
1336         push    %rbp
1337         push    %r12
1338         push    %r13
1339         push    %r14
1340         push    %r15
1341         pushfq
1342         sub     \$64,%rsp
1343
1344         mov     120($context),%rax      # pull context->Rax
1345         mov     248($context),%rbx      # pull context->Rip
1346
1347         mov     8($disp),%rsi           # disp->ImageBase
1348         mov     56($disp),%r11          # disp->HandlerData
1349
1350         mov     0(%r11),%r10d           # HandlerData[0]
1351         lea     (%rsi,%r10),%r10        # end of prologue label
1352         cmp     %r10,%rbx               # context->Rip<.Lbody
1353         jb      .Lin_prologue
1354
1355         mov     152($context),%rax      # pull context->Rsp
1356
1357         mov     4(%r11),%r10d           # HandlerData[1]
1358         lea     (%rsi,%r10),%r10        # epilogue label
1359         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1360         jae     .Lin_prologue
1361
1362         mov     `16*17`(%rax),%rax      # pull saved stack pointer
1363
1364         mov     -8(%rax),%rbx
1365         mov     -16(%rax),%rbp
1366         mov     %rbx,144($context)      # restore context->Rbx
1367         mov     %rbp,160($context)      # restore context->Rbp
1368
1369         lea     -24-10*16(%rax),%rsi
1370         lea     512($context),%rdi      # &context.Xmm6
1371         mov     \$20,%ecx
1372         .long   0xa548f3fc              # cld; rep movsq
1373
1374 .Lin_prologue:
1375         mov     8(%rax),%rdi
1376         mov     16(%rax),%rsi
1377         mov     %rax,152($context)      # restore context->Rsp
1378         mov     %rsi,168($context)      # restore context->Rsi
1379         mov     %rdi,176($context)      # restore context->Rdi
1380
1381         mov     40($disp),%rdi          # disp->ContextRecord
1382         mov     $context,%rsi           # context
1383         mov     \$154,%ecx              # sizeof(CONTEXT)
1384         .long   0xa548f3fc              # cld; rep movsq
1385
1386         mov     $disp,%rsi
1387         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1388         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1389         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1390         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1391         mov     40(%rsi),%r10           # disp->ContextRecord
1392         lea     56(%rsi),%r11           # &disp->HandlerData
1393         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1394         mov     %r10,32(%rsp)           # arg5
1395         mov     %r11,40(%rsp)           # arg6
1396         mov     %r12,48(%rsp)           # arg7
1397         mov     %rcx,56(%rsp)           # arg8, (NULL)
1398         call    *__imp_RtlVirtualUnwind(%rip)
1399
1400         mov     \$1,%eax                # ExceptionContinueSearch
1401         add     \$64,%rsp
1402         popfq
1403         pop     %r15
1404         pop     %r14
1405         pop     %r13
1406         pop     %r12
1407         pop     %rbp
1408         pop     %rbx
1409         pop     %rdi
1410         pop     %rsi
1411         ret
1412 .size   se_handler,.-se_handler
1413 ___
1414 $code.=<<___ if ($avx>1);
1415 .type   avx2_handler,\@abi-omnipotent
1416 .align  16
1417 avx2_handler:
1418         push    %rsi
1419         push    %rdi
1420         push    %rbx
1421         push    %rbp
1422         push    %r12
1423         push    %r13
1424         push    %r14
1425         push    %r15
1426         pushfq
1427         sub     \$64,%rsp
1428
1429         mov     120($context),%rax      # pull context->Rax
1430         mov     248($context),%rbx      # pull context->Rip
1431
1432         mov     8($disp),%rsi           # disp->ImageBase
1433         mov     56($disp),%r11          # disp->HandlerData
1434
1435         mov     0(%r11),%r10d           # HandlerData[0]
1436         lea     (%rsi,%r10),%r10        # end of prologue label
1437         cmp     %r10,%rbx               # context->Rip<body label
1438         jb      .Lin_prologue
1439
1440         mov     152($context),%rax      # pull context->Rsp
1441
1442         mov     4(%r11),%r10d           # HandlerData[1]
1443         lea     (%rsi,%r10),%r10        # epilogue label
1444         cmp     %r10,%rbx               # context->Rip>=epilogue label
1445         jae     .Lin_prologue
1446
1447         mov     `32*17`($context),%rax  # pull saved stack pointer
1448
1449         mov     -8(%rax),%rbx
1450         mov     -16(%rax),%rbp
1451         mov     -24(%rax),%r12
1452         mov     -32(%rax),%r13
1453         mov     -40(%rax),%r14
1454         mov     -48(%rax),%r15
1455         mov     %rbx,144($context)      # restore context->Rbx
1456         mov     %rbp,160($context)      # restore context->Rbp
1457         mov     %r12,216($context)      # restore cotnext->R12
1458         mov     %r13,224($context)      # restore cotnext->R13
1459         mov     %r14,232($context)      # restore cotnext->R14
1460         mov     %r15,240($context)      # restore cotnext->R15
1461
1462         lea     -56-10*16(%rax),%rsi
1463         lea     512($context),%rdi      # &context.Xmm6
1464         mov     \$20,%ecx
1465         .long   0xa548f3fc              # cld; rep movsq
1466
1467         jmp     .Lin_prologue
1468 .size   avx2_handler,.-avx2_handler
1469 ___
1470 $code.=<<___;
1471 .section        .pdata
1472 .align  4
1473         .rva    .LSEH_begin_sha1_multi_block
1474         .rva    .LSEH_end_sha1_multi_block
1475         .rva    .LSEH_info_sha1_multi_block
1476         .rva    .LSEH_begin_sha1_multi_block_shaext
1477         .rva    .LSEH_end_sha1_multi_block_shaext
1478         .rva    .LSEH_info_sha1_multi_block_shaext
1479 ___
1480 $code.=<<___ if ($avx);
1481         .rva    .LSEH_begin_sha1_multi_block_avx
1482         .rva    .LSEH_end_sha1_multi_block_avx
1483         .rva    .LSEH_info_sha1_multi_block_avx
1484 ___
1485 $code.=<<___ if ($avx>1);
1486         .rva    .LSEH_begin_sha1_multi_block_avx2
1487         .rva    .LSEH_end_sha1_multi_block_avx2
1488         .rva    .LSEH_info_sha1_multi_block_avx2
1489 ___
1490 $code.=<<___;
1491 .section        .xdata
1492 .align  8
1493 .LSEH_info_sha1_multi_block:
1494         .byte   9,0,0,0
1495         .rva    se_handler
1496         .rva    .Lbody,.Lepilogue                       # HandlerData[]
1497 .LSEH_info_sha1_multi_block_shaext:
1498         .byte   9,0,0,0
1499         .rva    se_handler
1500         .rva    .Lbody_shaext,.Lepilogue_shaext # HandlerData[]
1501 ___
1502 $code.=<<___ if ($avx);
1503 .LSEH_info_sha1_multi_block_avx:
1504         .byte   9,0,0,0
1505         .rva    se_handler
1506         .rva    .Lbody_avx,.Lepilogue_avx               # HandlerData[]
1507 ___
1508 $code.=<<___ if ($avx>1);
1509 .LSEH_info_sha1_multi_block_avx2:
1510         .byte   9,0,0,0
1511         .rva    avx2_handler
1512         .rva    .Lbody_avx2,.Lepilogue_avx2             # HandlerData[]
1513 ___
1514 }
1515 ####################################################################
1516
1517 sub rex {
1518   local *opcode=shift;
1519   my ($dst,$src)=@_;
1520   my $rex=0;
1521
1522     $rex|=0x04                  if ($dst>=8);
1523     $rex|=0x01                  if ($src>=8);
1524     unshift @opcode,$rex|0x40   if ($rex);
1525 }
1526
1527 sub sha1rnds4 {
1528     if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1529       my @opcode=(0x0f,0x3a,0xcc);
1530         rex(\@opcode,$3,$2);
1531         push @opcode,0xc0|($2&7)|(($3&7)<<3);           # ModR/M
1532         my $c=$1;
1533         push @opcode,$c=~/^0/?oct($c):$c;
1534         return ".byte\t".join(',',@opcode);
1535     } else {
1536         return "sha1rnds4\t".@_[0];
1537     }
1538 }
1539
1540 sub sha1op38 {
1541     my $instr = shift;
1542     my %opcodelet = (
1543                 "sha1nexte" => 0xc8,
1544                 "sha1msg1"  => 0xc9,
1545                 "sha1msg2"  => 0xca     );
1546
1547     if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1548       my @opcode=(0x0f,0x38);
1549         rex(\@opcode,$2,$1);
1550         push @opcode,$opcodelet{$instr};
1551         push @opcode,0xc0|($1&7)|(($2&7)<<3);           # ModR/M
1552         return ".byte\t".join(',',@opcode);
1553     } else {
1554         return $instr."\t".@_[0];
1555     }
1556 }
1557
1558 foreach (split("\n",$code)) {
1559         s/\`([^\`]*)\`/eval($1)/ge;
1560
1561         s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo                or
1562         s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo             or
1563
1564         s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go          or
1565         s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go         or
1566         s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go    or
1567         s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go        or
1568         s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go            or
1569         s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;
1570
1571         print $_,"\n";
1572 }
1573
1574 close STDOUT;