crypto/sha/asm/sha1-mb-x86_64.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9
  10 # Multi-buffer SHA1 procedure processes n buffers in parallel by
  11 # placing buffer data to designated lane of SIMD register. n is
  12 # naturally limited to 4 on pre-AVX2 processors and to 8 on
  13 # AVX2-capable processors such as Haswell.
  14 #
  15 #               this    +aesni(i)       sha1    aesni-sha1      gain(iv)
  16 # -------------------------------------------------------------------
  17 # Westmere(ii)  10.7/n  +1.28=3.96(n=4) 5.30    6.66            +68%
  18 # Atom(ii)      18.1/n  +3.93=8.46(n=4) 9.37    12.8            +51%
  19 # Sandy Bridge  (8.16   +5.15=13.3)/n   4.99    5.98            +80%
  20 # Ivy Bridge    (8.08   +5.14=13.2)/n   4.60    5.54            +68%
  21 # Haswell(iii)  (8.96   +5.00=14.0)/n   3.57    4.55            +160%
  22 # Bulldozer     (9.76   +5.76=15.5)/n   5.95    6.37            +64%
  23 #
  24 # (i)   multi-block CBC encrypt with 128-bit key;
  25 # (ii)  (HASH+AES)/n does not apply to Westmere for n>3 and Atom,
  26 #       because of lower AES-NI instruction throughput;
  27 # (iii) "this" is for n=8, when we gather twice as much data, result
  28 #       for n=4 is 8.00+4.44=12.4;
  29 # (iv)  presented improvement coefficients are asymptotic limits and
  30 #       in real-life application are somewhat lower, e.g. for 2KB
  31 #       fragments they range from 30% to 100% (on Haswell);
  32
  33 $flavour = shift;
  34 $output  = shift;
  35 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  36
  37 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  38
  39 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  40 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  41 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  42 die "can't locate x86_64-xlate.pl";
  43
  44 $avx=0;
  45
  46 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  47                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  48         $avx = ($1>=2.19) + ($1>=2.22);
  49 }
  50
  51 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  52            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  53         $avx = ($1>=2.09) + ($1>=2.10);
  54 }
  55
  56 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  57            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  58         $avx = ($1>=10) + ($1>=11);
  59 }
  60
  61 if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) {
  62         $avx = ($2>=3.0) + ($2>3.0);
  63 }
  64
  65 open OUT,"| \"$^X\" $xlate $flavour $output";
  66 *STDOUT=*OUT;
  67
  68 # void sha1_multi_block (
  69 #     struct {  unsigned int A[8];
  70 #               unsigned int B[8];
  71 #               unsigned int C[8];
  72 #               unsigned int D[8];
  73 #               unsigned int E[8];      } *ctx,
  74 #     struct {  void *ptr; int blocks;  } inp[8],
  75 #     int num);         /* 1 or 2 */
  76 #
  77 $ctx="%rdi";    # 1st arg
  78 $inp="%rsi";    # 2nd arg
  79 $num="%edx";
  80 @ptr=map("%r$_",(8..11));
  81 $Tbl="%rbp";
  82
  83 @V=($A,$B,$C,$D,$E)=map("%xmm$_",(0..4));
  84 ($t0,$t1,$t2,$t3,$tx)=map("%xmm$_",(5..9));
  85 @Xi=map("%xmm$_",(10..14));
  86 $K="%xmm15";
  87
  88 if (1) {
  89     # Atom-specific optimization aiming to eliminate pshufb with high
  90     # registers [and thus get rid of 48 cycles accumulated penalty]
  91     @Xi=map("%xmm$_",(0..4));
  92     ($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9));
  93     @V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14));
  94 }
  95
  96 $REG_SZ=16;
  97
  98 sub Xi_off {
  99 my $off = shift;
 100
 101     $off %= 16; $off *= $REG_SZ;
 102     $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
 103 }
 104
 105 sub BODY_00_19 {
 106 my ($i,$a,$b,$c,$d,$e)=@_;
 107 my $j=$i+1;
 108 my $k=$i+2;
 109
 110 $code.=<<___ if ($i==0);
 111         movd            (@ptr[0]),@Xi[0]
 112          lea            `16*4`(@ptr[0]),@ptr[0]
 113         movd            (@ptr[1]),@Xi[2]        # borrow @Xi[2]
 114          lea            `16*4`(@ptr[1]),@ptr[1]
 115         movd            (@ptr[2]),@Xi[3]        # borrow @Xi[3]
 116          lea            `16*4`(@ptr[2]),@ptr[2]
 117         movd            (@ptr[3]),@Xi[4]        # borrow @Xi[4]
 118          lea            `16*4`(@ptr[3]),@ptr[3]
 119         punpckldq       @Xi[3],@Xi[0]
 120          movd           `4*$j-16*4`(@ptr[0]),@Xi[1]
 121         punpckldq       @Xi[4],@Xi[2]
 122          movd           `4*$j-16*4`(@ptr[1]),$t3
 123         punpckldq       @Xi[2],@Xi[0]
 124          movd           `4*$j-16*4`(@ptr[2]),$t2
 125         pshufb          $tx,@Xi[0]
 126 ___
 127 $code.=<<___ if ($i<14);                        # just load input
 128          movd           `4*$j-16*4`(@ptr[3]),$t1
 129          punpckldq      $t2,@Xi[1]
 130         movdqa  $a,$t2
 131         paddd   $K,$e                           # e+=K_00_19
 132          punpckldq      $t1,$t3
 133         movdqa  $b,$t1
 134         movdqa  $b,$t0
 135         pslld   \$5,$t2
 136         pandn   $d,$t1
 137         pand    $c,$t0
 138          punpckldq      $t3,@Xi[1]
 139         movdqa  $a,$t3
 140
 141         movdqa  @Xi[0],`&Xi_off($i)`
 142         paddd   @Xi[0],$e                       # e+=X[i]
 143          movd           `4*$k-16*4`(@ptr[0]),@Xi[2]
 144         psrld   \$27,$t3
 145         pxor    $t1,$t0                         # Ch(b,c,d)
 146         movdqa  $b,$t1
 147
 148         por     $t3,$t2                         # rol(a,5)
 149          movd           `4*$k-16*4`(@ptr[1]),$t3
 150         pslld   \$30,$t1
 151         paddd   $t0,$e                          # e+=Ch(b,c,d)
 152
 153         psrld   \$2,$b
 154         paddd   $t2,$e                          # e+=rol(a,5)
 155          pshufb $tx,@Xi[1]
 156          movd           `4*$k-16*4`(@ptr[2]),$t2
 157         por     $t1,$b                          # b=rol(b,30)
 158 ___
 159 $code.=<<___ if ($i==14);                       # just load input
 160          movd           `4*$j-16*4`(@ptr[3]),$t1
 161          punpckldq      $t2,@Xi[1]
 162         movdqa  $a,$t2
 163         paddd   $K,$e                           # e+=K_00_19
 164          punpckldq      $t1,$t3
 165         movdqa  $b,$t1
 166         movdqa  $b,$t0
 167         pslld   \$5,$t2
 168          prefetcht0     63(@ptr[0])
 169         pandn   $d,$t1
 170         pand    $c,$t0
 171          punpckldq      $t3,@Xi[1]
 172         movdqa  $a,$t3
 173
 174         movdqa  @Xi[0],`&Xi_off($i)`
 175         paddd   @Xi[0],$e                       # e+=X[i]
 176         psrld   \$27,$t3
 177         pxor    $t1,$t0                         # Ch(b,c,d)
 178         movdqa  $b,$t1
 179          prefetcht0     63(@ptr[1])
 180
 181         por     $t3,$t2                         # rol(a,5)
 182         pslld   \$30,$t1
 183         paddd   $t0,$e                          # e+=Ch(b,c,d)
 184          prefetcht0     63(@ptr[2])
 185
 186         psrld   \$2,$b
 187         paddd   $t2,$e                          # e+=rol(a,5)
 188          pshufb $tx,@Xi[1]
 189          prefetcht0     63(@ptr[3])
 190         por     $t1,$b                          # b=rol(b,30)
 191 ___
 192 $code.=<<___ if ($i>=13 && $i<15);
 193         movdqa  `&Xi_off($j+2)`,@Xi[3]          # preload "X[2]"
 194 ___
 195 $code.=<<___ if ($i>=15);                       # apply Xupdate
 196         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 197         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 198
 199         movdqa  $a,$t2
 200          pxor   `&Xi_off($j+8)`,@Xi[1]
 201         paddd   $K,$e                           # e+=K_00_19
 202         movdqa  $b,$t1
 203         pslld   \$5,$t2
 204          pxor   @Xi[3],@Xi[1]
 205         movdqa  $b,$t0
 206         pandn   $d,$t1
 207          movdqa @Xi[1],$tx
 208         pand    $c,$t0
 209         movdqa  $a,$t3
 210          psrld  \$31,$tx
 211          paddd  @Xi[1],@Xi[1]
 212
 213         movdqa  @Xi[0],`&Xi_off($i)`
 214         paddd   @Xi[0],$e                       # e+=X[i]
 215         psrld   \$27,$t3
 216         pxor    $t1,$t0                         # Ch(b,c,d)
 217
 218         movdqa  $b,$t1
 219         por     $t3,$t2                         # rol(a,5)
 220         pslld   \$30,$t1
 221         paddd   $t0,$e                          # e+=Ch(b,c,d)
 222
 223         psrld   \$2,$b
 224         paddd   $t2,$e                          # e+=rol(a,5)
 225          por    $tx,@Xi[1]                      # rol   \$1,@Xi[1]
 226         por     $t1,$b                          # b=rol(b,30)
 227 ___
 228 push(@Xi,shift(@Xi));
 229 }
 230
 231 sub BODY_20_39 {
 232 my ($i,$a,$b,$c,$d,$e)=@_;
 233 my $j=$i+1;
 234
 235 $code.=<<___ if ($i<79);
 236         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 237         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 238
 239         movdqa  $a,$t2
 240         movdqa  $d,$t0
 241          pxor   `&Xi_off($j+8)`,@Xi[1]
 242         paddd   $K,$e                           # e+=K_20_39
 243         pslld   \$5,$t2
 244         pxor    $b,$t0
 245
 246         movdqa  $a,$t3
 247 ___
 248 $code.=<<___ if ($i<72);
 249         movdqa  @Xi[0],`&Xi_off($i)`
 250 ___
 251 $code.=<<___ if ($i<79);
 252         paddd   @Xi[0],$e                       # e+=X[i]
 253          pxor   @Xi[3],@Xi[1]
 254         psrld   \$27,$t3
 255         pxor    $c,$t0                          # Parity(b,c,d)
 256         movdqa  $b,$t1
 257
 258         pslld   \$30,$t1
 259          movdqa @Xi[1],$tx
 260         por     $t3,$t2                         # rol(a,5)
 261          psrld  \$31,$tx
 262         paddd   $t0,$e                          # e+=Parity(b,c,d)
 263          paddd  @Xi[1],@Xi[1]
 264
 265         psrld   \$2,$b
 266         paddd   $t2,$e                          # e+=rol(a,5)
 267          por    $tx,@Xi[1]                      # rol(@Xi[1],1)
 268         por     $t1,$b                          # b=rol(b,30)
 269 ___
 270 $code.=<<___ if ($i==79);
 271         movdqa  $a,$t2
 272         paddd   $K,$e                           # e+=K_20_39
 273         movdqa  $d,$t0
 274         pslld   \$5,$t2
 275         pxor    $b,$t0
 276
 277         movdqa  $a,$t3
 278         paddd   @Xi[0],$e                       # e+=X[i]
 279         psrld   \$27,$t3
 280         movdqa  $b,$t1
 281         pxor    $c,$t0                          # Parity(b,c,d)
 282
 283         pslld   \$30,$t1
 284         por     $t3,$t2                         # rol(a,5)
 285         paddd   $t0,$e                          # e+=Parity(b,c,d)
 286
 287         psrld   \$2,$b
 288         paddd   $t2,$e                          # e+=rol(a,5)
 289         por     $t1,$b                          # b=rol(b,30)
 290 ___
 291 push(@Xi,shift(@Xi));
 292 }
 293
 294 sub BODY_40_59 {
 295 my ($i,$a,$b,$c,$d,$e)=@_;
 296 my $j=$i+1;
 297
 298 $code.=<<___;
 299         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 300         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 301
 302         movdqa  $a,$t2
 303         movdqa  $d,$t1
 304          pxor   `&Xi_off($j+8)`,@Xi[1]
 305         pxor    @Xi[3],@Xi[1]
 306         paddd   $K,$e                           # e+=K_40_59
 307         pslld   \$5,$t2
 308         movdqa  $a,$t3
 309         pand    $c,$t1
 310
 311         movdqa  $d,$t0
 312          movdqa @Xi[1],$tx
 313         psrld   \$27,$t3
 314         paddd   $t1,$e
 315         pxor    $c,$t0
 316
 317         movdqa  @Xi[0],`&Xi_off($i)`
 318         paddd   @Xi[0],$e                       # e+=X[i]
 319         por     $t3,$t2                         # rol(a,5)
 320          psrld  \$31,$tx
 321         pand    $b,$t0
 322         movdqa  $b,$t1
 323
 324         pslld   \$30,$t1
 325          paddd  @Xi[1],@Xi[1]
 326         paddd   $t0,$e                          # e+=Maj(b,d,c)
 327
 328         psrld   \$2,$b
 329         paddd   $t2,$e                          # e+=rol(a,5)
 330          por    $tx,@Xi[1]                      # rol(@X[1],1)
 331         por     $t1,$b                          # b=rol(b,30)
 332 ___
 333 push(@Xi,shift(@Xi));
 334 }
 335
 336 $code.=<<___;
 337 .text
 338
 339 .extern OPENSSL_ia32cap_P
 340
 341 .globl  sha1_multi_block
 342 .type   sha1_multi_block,\@function,3
 343 .align  32
 344 sha1_multi_block:
 345         mov     OPENSSL_ia32cap_P+4(%rip),%rcx
 346         bt      \$61,%rcx                       # check SHA bit
 347         jc      _shaext_shortcut
 348 ___
 349 $code.=<<___ if ($avx);
 350         test    \$`1<<28`,%ecx
 351         jnz     _avx_shortcut
 352 ___
 353 $code.=<<___;
 354         mov     %rsp,%rax
 355         push    %rbx
 356         push    %rbp
 357 ___
 358 $code.=<<___ if ($win64);
 359         lea     -0xa8(%rsp),%rsp
 360         movaps  %xmm6,(%rsp)
 361         movaps  %xmm7,0x10(%rsp)
 362         movaps  %xmm8,0x20(%rsp)
 363         movaps  %xmm9,0x30(%rsp)
 364         movaps  %xmm10,-0x78(%rax)
 365         movaps  %xmm11,-0x68(%rax)
 366         movaps  %xmm12,-0x58(%rax)
 367         movaps  %xmm13,-0x48(%rax)
 368         movaps  %xmm14,-0x38(%rax)
 369         movaps  %xmm15,-0x28(%rax)
 370 ___
 371 $code.=<<___;
 372         sub     \$`$REG_SZ*18`,%rsp
 373         and     \$-256,%rsp
 374         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 375 .Lbody:
 376         lea     K_XX_XX(%rip),$Tbl
 377         lea     `$REG_SZ*16`(%rsp),%rbx
 378
 379 .Loop_grande:
 380         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
 381         xor     $num,$num
 382 ___
 383 for($i=0;$i<4;$i++) {
 384     $code.=<<___;
 385         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 386         mov     `16*$i+8`($inp),%ecx            # number of blocks
 387         cmp     $num,%ecx
 388         cmovg   %ecx,$num                       # find maximum
 389         test    %ecx,%ecx
 390         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 391         cmovle  $Tbl,@ptr[$i]                   # cancel input
 392 ___
 393 }
 394 $code.=<<___;
 395         test    $num,$num
 396         jz      .Ldone
 397
 398         movdqu  0x00($ctx),$A                   # load context
 399          lea    128(%rsp),%rax
 400         movdqu  0x20($ctx),$B
 401         movdqu  0x40($ctx),$C
 402         movdqu  0x60($ctx),$D
 403         movdqu  0x80($ctx),$E
 404         movdqa  0x60($Tbl),$tx                  # pbswap_mask
 405         movdqa  -0x20($Tbl),$K                  # K_00_19
 406         jmp     .Loop
 407
 408 .align  32
 409 .Loop:
 410 ___
 411 for($i=0;$i<20;$i++)    { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
 412 $code.="        movdqa  0x00($Tbl),$K\n";       # K_20_39
 413 for(;$i<40;$i++)        { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
 414 $code.="        movdqa  0x20($Tbl),$K\n";       # K_40_59
 415 for(;$i<60;$i++)        { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
 416 $code.="        movdqa  0x40($Tbl),$K\n";       # K_60_79
 417 for(;$i<80;$i++)        { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
 418 $code.=<<___;
 419         movdqa  (%rbx),@Xi[0]                   # pull counters
 420         mov     \$1,%ecx
 421         cmp     4*0(%rbx),%ecx                  # examinte counters
 422         pxor    $t2,$t2
 423         cmovge  $Tbl,@ptr[0]                    # cancel input
 424         cmp     4*1(%rbx),%ecx
 425         movdqa  @Xi[0],@Xi[1]
 426         cmovge  $Tbl,@ptr[1]
 427         cmp     4*2(%rbx),%ecx
 428         pcmpgtd $t2,@Xi[1]                      # mask value
 429         cmovge  $Tbl,@ptr[2]
 430         cmp     4*3(%rbx),%ecx
 431         paddd   @Xi[1],@Xi[0]                   # counters--
 432         cmovge  $Tbl,@ptr[3]
 433
 434         movdqu  0x00($ctx),$t0
 435         pand    @Xi[1],$A
 436         movdqu  0x20($ctx),$t1
 437         pand    @Xi[1],$B
 438         paddd   $t0,$A
 439         movdqu  0x40($ctx),$t2
 440         pand    @Xi[1],$C
 441         paddd   $t1,$B
 442         movdqu  0x60($ctx),$t3
 443         pand    @Xi[1],$D
 444         paddd   $t2,$C
 445         movdqu  0x80($ctx),$tx
 446         pand    @Xi[1],$E
 447         movdqu  $A,0x00($ctx)
 448         paddd   $t3,$D
 449         movdqu  $B,0x20($ctx)
 450         paddd   $tx,$E
 451         movdqu  $C,0x40($ctx)
 452         movdqu  $D,0x60($ctx)
 453         movdqu  $E,0x80($ctx)
 454
 455         movdqa  @Xi[0],(%rbx)                   # save counters
 456         movdqa  0x60($Tbl),$tx                  # pbswap_mask
 457         movdqa  -0x20($Tbl),$K                  # K_00_19
 458         dec     $num
 459         jnz     .Loop
 460
 461         mov     `$REG_SZ*17+8`(%rsp),$num
 462         lea     $REG_SZ($ctx),$ctx
 463         lea     `16*$REG_SZ/4`($inp),$inp
 464         dec     $num
 465         jnz     .Loop_grande
 466
 467 .Ldone:
 468         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
 469 ___
 470 $code.=<<___ if ($win64);
 471         movaps  -0xb8(%rax),%xmm6
 472         movaps  -0xa8(%rax),%xmm7
 473         movaps  -0x98(%rax),%xmm8
 474         movaps  -0x88(%rax),%xmm9
 475         movaps  -0x78(%rax),%xmm10
 476         movaps  -0x68(%rax),%xmm11
 477         movaps  -0x58(%rax),%xmm12
 478         movaps  -0x48(%rax),%xmm13
 479         movaps  -0x38(%rax),%xmm14
 480         movaps  -0x28(%rax),%xmm15
 481 ___
 482 $code.=<<___;
 483         mov     -16(%rax),%rbp
 484         mov     -8(%rax),%rbx
 485         lea     (%rax),%rsp
 486 .Lepilogue:
 487         ret
 488 .size   sha1_multi_block,.-sha1_multi_block
 489 ___
 490                                                 {{{
 491 my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10));
 492 my @MSG0=map("%xmm$_",(4..7));
 493 my @MSG1=map("%xmm$_",(11..14));
 494
 495 $code.=<<___;
 496 .type   sha1_multi_block_shaext,\@function,3
 497 .align  32
 498 sha1_multi_block_shaext:
 499 _shaext_shortcut:
 500         mov     %rsp,%rax
 501         push    %rbx
 502         push    %rbp
 503 ___
 504 $code.=<<___ if ($win64);
 505         lea     -0xa8(%rsp),%rsp
 506         movaps  %xmm6,(%rsp)
 507         movaps  %xmm7,0x10(%rsp)
 508         movaps  %xmm8,0x20(%rsp)
 509         movaps  %xmm9,0x30(%rsp)
 510         movaps  %xmm10,-0x78(%rax)
 511         movaps  %xmm11,-0x68(%rax)
 512         movaps  %xmm12,-0x58(%rax)
 513         movaps  %xmm13,-0x48(%rax)
 514         movaps  %xmm14,-0x38(%rax)
 515         movaps  %xmm15,-0x28(%rax)
 516 ___
 517 $code.=<<___;
 518         sub     \$`$REG_SZ*18`,%rsp
 519         shl     \$1,$num                        # we process pair at a time
 520         and     \$-256,%rsp
 521         lea     0x40($ctx),$ctx                 # size optimization
 522         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 523 .Lbody_shaext:
 524         lea     `$REG_SZ*16`(%rsp),%rbx
 525         movdqa  K_XX_XX+0x80(%rip),$BSWAP       # byte-n-word swap
 526
 527 .Loop_grande_shaext:
 528         mov     $num,`$REG_SZ*17+8`(%rsp)       # orignal $num
 529         xor     $num,$num
 530 ___
 531 for($i=0;$i<2;$i++) {
 532     $code.=<<___;
 533         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 534         mov     `16*$i+8`($inp),%ecx            # number of blocks
 535         cmp     $num,%ecx
 536         cmovg   %ecx,$num                       # find maximum
 537         test    %ecx,%ecx
 538         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 539         cmovle  %rsp,@ptr[$i]                   # cancel input
 540 ___
 541 }
 542 $code.=<<___;
 543         test    $num,$num
 544         jz      .Ldone_shaext
 545
 546         movq            0x00-0x40($ctx),$ABCD0  # a1.a0
 547         movq            0x20-0x40($ctx),@MSG0[0]# b1.b0
 548         movq            0x40-0x40($ctx),@MSG0[1]# c1.c0
 549         movq            0x60-0x40($ctx),@MSG0[2]# d1.d0
 550         movq            0x80-0x40($ctx),@MSG0[3]# e1.e0
 551
 552         punpckldq       @MSG0[0],$ABCD0         # b1.a1.b0.a0
 553         punpckldq       @MSG0[2],@MSG0[1]       # d1.c1.d0.c0
 554
 555         movdqa          $ABCD0,$ABCD1
 556         punpcklqdq      @MSG0[1],$ABCD0         # d0.c0.b0.a0
 557         punpckhqdq      @MSG0[1],$ABCD1         # d1.c1.b1.a1
 558
 559         pshufd          \$0b00111111,@MSG0[3],$E0
 560         pshufd          \$0b01111111,@MSG0[3],$E1
 561         pshufd          \$0b00011011,$ABCD0,$ABCD0
 562         pshufd          \$0b00011011,$ABCD1,$ABCD1
 563         jmp             .Loop_shaext
 564
 565 .align  32
 566 .Loop_shaext:
 567         movdqu          0x00(@ptr[0]),@MSG0[0]
 568          movdqu         0x00(@ptr[1]),@MSG1[0]
 569         movdqu          0x10(@ptr[0]),@MSG0[1]
 570          movdqu         0x10(@ptr[1]),@MSG1[1]
 571         movdqu          0x20(@ptr[0]),@MSG0[2]
 572         pshufb          $BSWAP,@MSG0[0]
 573          movdqu         0x20(@ptr[1]),@MSG1[2]
 574          pshufb         $BSWAP,@MSG1[0]
 575         movdqu          0x30(@ptr[0]),@MSG0[3]
 576         lea             0x40(@ptr[0]),@ptr[0]
 577         pshufb          $BSWAP,@MSG0[1]
 578          movdqu         0x30(@ptr[1]),@MSG1[3]
 579          lea            0x40(@ptr[1]),@ptr[1]
 580          pshufb         $BSWAP,@MSG1[1]
 581
 582         movdqa          $E0,0x50(%rsp)          # offload
 583         paddd           @MSG0[0],$E0
 584          movdqa         $E1,0x70(%rsp)
 585          paddd          @MSG1[0],$E1
 586         movdqa          $ABCD0,0x40(%rsp)       # offload
 587         movdqa          $ABCD0,$E0_
 588          movdqa         $ABCD1,0x60(%rsp)
 589          movdqa         $ABCD1,$E1_
 590         sha1rnds4       \$0,$E0,$ABCD0          # 0-3
 591         sha1nexte       @MSG0[1],$E0_
 592          sha1rnds4      \$0,$E1,$ABCD1          # 0-3
 593          sha1nexte      @MSG1[1],$E1_
 594         pshufb          $BSWAP,@MSG0[2]
 595         prefetcht0      127(@ptr[0])
 596         sha1msg1        @MSG0[1],@MSG0[0]
 597          pshufb         $BSWAP,@MSG1[2]
 598          prefetcht0     127(@ptr[1])
 599          sha1msg1       @MSG1[1],@MSG1[0]
 600
 601         pshufb          $BSWAP,@MSG0[3]
 602         movdqa          $ABCD0,$E0
 603          pshufb         $BSWAP,@MSG1[3]
 604          movdqa         $ABCD1,$E1
 605         sha1rnds4       \$0,$E0_,$ABCD0         # 4-7
 606         sha1nexte       @MSG0[2],$E0
 607          sha1rnds4      \$0,$E1_,$ABCD1         # 4-7
 608          sha1nexte      @MSG1[2],$E1
 609         pxor            @MSG0[2],@MSG0[0]
 610         sha1msg1        @MSG0[2],@MSG0[1]
 611          pxor           @MSG1[2],@MSG1[0]
 612          sha1msg1       @MSG1[2],@MSG1[1]
 613 ___
 614 for($i=2;$i<20-4;$i++) {
 615 $code.=<<___;
 616         movdqa          $ABCD0,$E0_
 617          movdqa         $ABCD1,$E1_
 618         sha1rnds4       \$`int($i/5)`,$E0,$ABCD0        # 8-11
 619         sha1nexte       @MSG0[3],$E0_
 620          sha1rnds4      \$`int($i/5)`,$E1,$ABCD1        # 8-11
 621          sha1nexte      @MSG1[3],$E1_
 622         sha1msg2        @MSG0[3],@MSG0[0]
 623          sha1msg2       @MSG1[3],@MSG1[0]
 624         pxor            @MSG0[3],@MSG0[1]
 625         sha1msg1        @MSG0[3],@MSG0[2]
 626          pxor           @MSG1[3],@MSG1[1]
 627          sha1msg1       @MSG1[3],@MSG1[2]
 628 ___
 629         ($E0,$E0_)=($E0_,$E0);          ($E1,$E1_)=($E1_,$E1);
 630         push(@MSG0,shift(@MSG0));       push(@MSG1,shift(@MSG1));
 631 }
 632 $code.=<<___;
 633         movdqa          $ABCD0,$E0_
 634          movdqa         $ABCD1,$E1_
 635         sha1rnds4       \$3,$E0,$ABCD0          # 64-67
 636         sha1nexte       @MSG0[3],$E0_
 637          sha1rnds4      \$3,$E1,$ABCD1          # 64-67
 638          sha1nexte      @MSG1[3],$E1_
 639         sha1msg2        @MSG0[3],@MSG0[0]
 640          sha1msg2       @MSG1[3],@MSG1[0]
 641         pxor            @MSG0[3],@MSG0[1]
 642          pxor           @MSG1[3],@MSG1[1]
 643
 644         mov             \$1,%ecx
 645         pxor            @MSG0[2],@MSG0[2]       # zero
 646         cmp             4*0(%rbx),%ecx          # examine counters
 647         cmovge          %rsp,@ptr[0]            # cancel input
 648
 649         movdqa          $ABCD0,$E0
 650          movdqa         $ABCD1,$E1
 651         sha1rnds4       \$3,$E0_,$ABCD0         # 68-71
 652         sha1nexte       @MSG0[0],$E0
 653          sha1rnds4      \$3,$E1_,$ABCD1         # 68-71
 654          sha1nexte      @MSG1[0],$E1
 655         sha1msg2        @MSG0[0],@MSG0[1]
 656          sha1msg2       @MSG1[0],@MSG1[1]
 657
 658         cmp             4*1(%rbx),%ecx
 659         cmovge          %rsp,@ptr[1]
 660         movq            (%rbx),@MSG0[0]         # pull counters
 661
 662         movdqa          $ABCD0,$E0_
 663          movdqa         $ABCD1,$E1_
 664         sha1rnds4       \$3,$E0,$ABCD0          # 72-75
 665         sha1nexte       @MSG0[1],$E0_
 666          sha1rnds4      \$3,$E1,$ABCD1          # 72-75
 667          sha1nexte      @MSG1[1],$E1_
 668
 669         pshufd          \$0x00,@MSG0[0],@MSG1[2]
 670         pshufd          \$0x55,@MSG0[0],@MSG1[3]
 671         movdqa          @MSG0[0],@MSG0[1]
 672         pcmpgtd         @MSG0[2],@MSG1[2]
 673         pcmpgtd         @MSG0[2],@MSG1[3]
 674
 675         movdqa          $ABCD0,$E0
 676          movdqa         $ABCD1,$E1
 677         sha1rnds4       \$3,$E0_,$ABCD0         # 76-79
 678         sha1nexte       $MSG0[2],$E0
 679          sha1rnds4      \$3,$E1_,$ABCD1         # 76-79
 680          sha1nexte      $MSG0[2],$E1
 681
 682         pcmpgtd         @MSG0[2],@MSG0[1]       # counter mask
 683         pand            @MSG1[2],$ABCD0
 684         pand            @MSG1[2],$E0
 685          pand           @MSG1[3],$ABCD1
 686          pand           @MSG1[3],$E1
 687         paddd           @MSG0[1],@MSG0[0]       # counters--
 688
 689         paddd           0x40(%rsp),$ABCD0
 690         paddd           0x50(%rsp),$E0
 691          paddd          0x60(%rsp),$ABCD1
 692          paddd          0x70(%rsp),$E1
 693
 694         movq            @MSG0[0],(%rbx)         # save counters
 695         dec             $num
 696         jnz             .Loop_shaext
 697
 698         mov             `$REG_SZ*17+8`(%rsp),$num
 699
 700         pshufd          \$0b00011011,$ABCD0,$ABCD0
 701         pshufd          \$0b00011011,$ABCD1,$ABCD1
 702
 703         movdqa          $ABCD0,@MSG0[0]
 704         punpckldq       $ABCD1,$ABCD0           # b1.b0.a1.a0
 705         punpckhdq       $ABCD1,@MSG0[0]         # d1.d0.c1.c0
 706         punpckhdq       $E1,$E0                 # e1.e0.xx.xx
 707         movq            $ABCD0,0x00-0x40($ctx)  # a1.a0
 708         psrldq          \$8,$ABCD0
 709         movq            @MSG0[0],0x40-0x40($ctx)# c1.c0
 710         psrldq          \$8,@MSG0[0]
 711         movq            $ABCD0,0x20-0x40($ctx)  # b1.b0
 712         psrldq          \$8,$E0
 713         movq            @MSG0[0],0x60-0x40($ctx)# d1.d0
 714         movq            $E0,0x80-0x40($ctx)     # e1.e0
 715
 716         lea     `$REG_SZ/2`($ctx),$ctx
 717         lea     `16*2`($inp),$inp
 718         dec     $num
 719         jnz     .Loop_grande_shaext
 720
 721 .Ldone_shaext:
 722         #mov    `$REG_SZ*17`(%rsp),%rax         # original %rsp
 723 ___
 724 $code.=<<___ if ($win64);
 725         movaps  -0xb8(%rax),%xmm6
 726         movaps  -0xa8(%rax),%xmm7
 727         movaps  -0x98(%rax),%xmm8
 728         movaps  -0x88(%rax),%xmm9
 729         movaps  -0x78(%rax),%xmm10
 730         movaps  -0x68(%rax),%xmm11
 731         movaps  -0x58(%rax),%xmm12
 732         movaps  -0x48(%rax),%xmm13
 733         movaps  -0x38(%rax),%xmm14
 734         movaps  -0x28(%rax),%xmm15
 735 ___
 736 $code.=<<___;
 737         mov     -16(%rax),%rbp
 738         mov     -8(%rax),%rbx
 739         lea     (%rax),%rsp
 740 .Lepilogue_shaext:
 741         ret
 742 .size   sha1_multi_block_shaext,.-sha1_multi_block_shaext
 743 ___
 744                                                 }}}
 745
 746                                                 if ($avx) {{{
 747 sub BODY_00_19_avx {
 748 my ($i,$a,$b,$c,$d,$e)=@_;
 749 my $j=$i+1;
 750 my $k=$i+2;
 751 my $vpack = $REG_SZ==16 ? "vpunpckldq" : "vinserti128";
 752 my $ptr_n = $REG_SZ==16 ? @ptr[1] : @ptr[4];
 753
 754 $code.=<<___ if ($i==0 && $REG_SZ==16);
 755         vmovd           (@ptr[0]),@Xi[0]
 756          lea            `16*4`(@ptr[0]),@ptr[0]
 757         vmovd           (@ptr[1]),@Xi[2]        # borrow Xi[2]
 758          lea            `16*4`(@ptr[1]),@ptr[1]
 759         vpinsrd         \$1,(@ptr[2]),@Xi[0],@Xi[0]
 760          lea            `16*4`(@ptr[2]),@ptr[2]
 761         vpinsrd         \$1,(@ptr[3]),@Xi[2],@Xi[2]
 762          lea            `16*4`(@ptr[3]),@ptr[3]
 763          vmovd          `4*$j-16*4`(@ptr[0]),@Xi[1]
 764         vpunpckldq      @Xi[2],@Xi[0],@Xi[0]
 765          vmovd          `4*$j-16*4`($ptr_n),$t3
 766         vpshufb         $tx,@Xi[0],@Xi[0]
 767 ___
 768 $code.=<<___ if ($i<15 && $REG_SZ==16);         # just load input
 769          vpinsrd        \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
 770          vpinsrd        \$1,`4*$j-16*4`(@ptr[3]),$t3,$t3
 771 ___
 772 $code.=<<___ if ($i==0 && $REG_SZ==32);
 773         vmovd           (@ptr[0]),@Xi[0]
 774          lea            `16*4`(@ptr[0]),@ptr[0]
 775         vmovd           (@ptr[4]),@Xi[2]        # borrow Xi[2]
 776          lea            `16*4`(@ptr[4]),@ptr[4]
 777         vmovd           (@ptr[1]),$t2
 778          lea            `16*4`(@ptr[1]),@ptr[1]
 779         vmovd           (@ptr[5]),$t1
 780          lea            `16*4`(@ptr[5]),@ptr[5]
 781         vpinsrd         \$1,(@ptr[2]),@Xi[0],@Xi[0]
 782          lea            `16*4`(@ptr[2]),@ptr[2]
 783         vpinsrd         \$1,(@ptr[6]),@Xi[2],@Xi[2]
 784          lea            `16*4`(@ptr[6]),@ptr[6]
 785         vpinsrd         \$1,(@ptr[3]),$t2,$t2
 786          lea            `16*4`(@ptr[3]),@ptr[3]
 787         vpunpckldq      $t2,@Xi[0],@Xi[0]
 788         vpinsrd         \$1,(@ptr[7]),$t1,$t1
 789          lea            `16*4`(@ptr[7]),@ptr[7]
 790         vpunpckldq      $t1,@Xi[2],@Xi[2]
 791          vmovd          `4*$j-16*4`(@ptr[0]),@Xi[1]
 792         vinserti128     @Xi[2],@Xi[0],@Xi[0]
 793          vmovd          `4*$j-16*4`($ptr_n),$t3
 794         vpshufb         $tx,@Xi[0],@Xi[0]
 795 ___
 796 $code.=<<___ if ($i<15 && $REG_SZ==32);         # just load input
 797          vmovd          `4*$j-16*4`(@ptr[1]),$t2
 798          vmovd          `4*$j-16*4`(@ptr[5]),$t1
 799          vpinsrd        \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
 800          vpinsrd        \$1,`4*$j-16*4`(@ptr[6]),$t3,$t3
 801          vpinsrd        \$1,`4*$j-16*4`(@ptr[3]),$t2,$t2
 802          vpunpckldq     $t2,@Xi[1],@Xi[1]
 803          vpinsrd        \$1,`4*$j-16*4`(@ptr[7]),$t1,$t1
 804          vpunpckldq     $t1,$t3,$t3
 805 ___
 806 $code.=<<___ if ($i<14);
 807         vpaddd  $K,$e,$e                        # e+=K_00_19
 808         vpslld  \$5,$a,$t2
 809         vpandn  $d,$b,$t1
 810         vpand   $c,$b,$t0
 811
 812         vmovdqa @Xi[0],`&Xi_off($i)`
 813         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 814          $vpack         $t3,@Xi[1],@Xi[1]
 815         vpsrld  \$27,$a,$t3
 816         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 817          vmovd          `4*$k-16*4`(@ptr[0]),@Xi[2]
 818
 819         vpslld  \$30,$b,$t1
 820         vpor    $t3,$t2,$t2                     # rol(a,5)
 821          vmovd          `4*$k-16*4`($ptr_n),$t3
 822         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 823
 824         vpsrld  \$2,$b,$b
 825         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 826          vpshufb        $tx,@Xi[1],@Xi[1]
 827         vpor    $t1,$b,$b                       # b=rol(b,30)
 828 ___
 829 $code.=<<___ if ($i==14);
 830         vpaddd  $K,$e,$e                        # e+=K_00_19
 831          prefetcht0     63(@ptr[0])
 832         vpslld  \$5,$a,$t2
 833         vpandn  $d,$b,$t1
 834         vpand   $c,$b,$t0
 835
 836         vmovdqa @Xi[0],`&Xi_off($i)`
 837         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 838          $vpack         $t3,@Xi[1],@Xi[1]
 839         vpsrld  \$27,$a,$t3
 840          prefetcht0     63(@ptr[1])
 841         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 842
 843         vpslld  \$30,$b,$t1
 844         vpor    $t3,$t2,$t2                     # rol(a,5)
 845          prefetcht0     63(@ptr[2])
 846         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 847
 848         vpsrld  \$2,$b,$b
 849         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 850          prefetcht0     63(@ptr[3])
 851          vpshufb        $tx,@Xi[1],@Xi[1]
 852         vpor    $t1,$b,$b                       # b=rol(b,30)
 853 ___
 854 $code.=<<___ if ($i>=13 && $i<15);
 855         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # preload "X[2]"
 856 ___
 857 $code.=<<___ if ($i>=15);                       # apply Xupdate
 858         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 859         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 860
 861         vpaddd  $K,$e,$e                        # e+=K_00_19
 862         vpslld  \$5,$a,$t2
 863         vpandn  $d,$b,$t1
 864          `"prefetcht0   63(@ptr[4])"            if ($i==15 && $REG_SZ==32)`
 865         vpand   $c,$b,$t0
 866
 867         vmovdqa @Xi[0],`&Xi_off($i)`
 868         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 869          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 870         vpsrld  \$27,$a,$t3
 871         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 872          vpxor  @Xi[3],@Xi[1],@Xi[1]
 873          `"prefetcht0   63(@ptr[5])"            if ($i==15 && $REG_SZ==32)`
 874
 875         vpslld  \$30,$b,$t1
 876         vpor    $t3,$t2,$t2                     # rol(a,5)
 877         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 878          `"prefetcht0   63(@ptr[6])"            if ($i==15 && $REG_SZ==32)`
 879          vpsrld \$31,@Xi[1],$tx
 880          vpaddd @Xi[1],@Xi[1],@Xi[1]
 881
 882         vpsrld  \$2,$b,$b
 883          `"prefetcht0   63(@ptr[7])"            if ($i==15 && $REG_SZ==32)`
 884         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 885          vpor   $tx,@Xi[1],@Xi[1]               # rol   \$1,@Xi[1]
 886         vpor    $t1,$b,$b                       # b=rol(b,30)
 887 ___
 888 push(@Xi,shift(@Xi));
 889 }
 890
 891 sub BODY_20_39_avx {
 892 my ($i,$a,$b,$c,$d,$e)=@_;
 893 my $j=$i+1;
 894
 895 $code.=<<___ if ($i<79);
 896         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 897         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 898
 899         vpslld  \$5,$a,$t2
 900         vpaddd  $K,$e,$e                        # e+=K_20_39
 901         vpxor   $b,$d,$t0
 902 ___
 903 $code.=<<___ if ($i<72);
 904         vmovdqa @Xi[0],`&Xi_off($i)`
 905 ___
 906 $code.=<<___ if ($i<79);
 907         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 908          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 909         vpsrld  \$27,$a,$t3
 910         vpxor   $c,$t0,$t0                      # Parity(b,c,d)
 911          vpxor  @Xi[3],@Xi[1],@Xi[1]
 912
 913         vpslld  \$30,$b,$t1
 914         vpor    $t3,$t2,$t2                     # rol(a,5)
 915         vpaddd  $t0,$e,$e                       # e+=Parity(b,c,d)
 916          vpsrld \$31,@Xi[1],$tx
 917          vpaddd @Xi[1],@Xi[1],@Xi[1]
 918
 919         vpsrld  \$2,$b,$b
 920         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 921          vpor   $tx,@Xi[1],@Xi[1]               # rol(@Xi[1],1)
 922         vpor    $t1,$b,$b                       # b=rol(b,30)
 923 ___
 924 $code.=<<___ if ($i==79);
 925         vpslld  \$5,$a,$t2
 926         vpaddd  $K,$e,$e                        # e+=K_20_39
 927         vpxor   $b,$d,$t0
 928
 929         vpsrld  \$27,$a,$t3
 930         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 931         vpxor   $c,$t0,$t0                      # Parity(b,c,d)
 932
 933         vpslld  \$30,$b,$t1
 934         vpor    $t3,$t2,$t2                     # rol(a,5)
 935         vpaddd  $t0,$e,$e                       # e+=Parity(b,c,d)
 936
 937         vpsrld  \$2,$b,$b
 938         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 939         vpor    $t1,$b,$b                       # b=rol(b,30)
 940 ___
 941 push(@Xi,shift(@Xi));
 942 }
 943
 944 sub BODY_40_59_avx {
 945 my ($i,$a,$b,$c,$d,$e)=@_;
 946 my $j=$i+1;
 947
 948 $code.=<<___;
 949         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 950         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 951
 952         vpaddd  $K,$e,$e                        # e+=K_40_59
 953         vpslld  \$5,$a,$t2
 954         vpand   $c,$d,$t1
 955          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 956
 957         vpaddd  $t1,$e,$e
 958         vpsrld  \$27,$a,$t3
 959         vpxor   $c,$d,$t0
 960          vpxor  @Xi[3],@Xi[1],@Xi[1]
 961
 962         vmovdqu @Xi[0],`&Xi_off($i)`
 963         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 964         vpor    $t3,$t2,$t2                     # rol(a,5)
 965          vpsrld \$31,@Xi[1],$tx
 966         vpand   $b,$t0,$t0
 967          vpaddd @Xi[1],@Xi[1],@Xi[1]
 968
 969         vpslld  \$30,$b,$t1
 970         vpaddd  $t0,$e,$e                       # e+=Maj(b,d,c)
 971
 972         vpsrld  \$2,$b,$b
 973         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 974          vpor   $tx,@Xi[1],@Xi[1]               # rol(@X[1],1)
 975         vpor    $t1,$b,$b                       # b=rol(b,30)
 976 ___
 977 push(@Xi,shift(@Xi));
 978 }
 979
 980 $code.=<<___;
 981 .type   sha1_multi_block_avx,\@function,3
 982 .align  32
 983 sha1_multi_block_avx:
 984 _avx_shortcut:
 985 ___
 986 $code.=<<___ if ($avx>1);
 987         shr     \$32,%rcx
 988         cmp     \$2,$num
 989         jb      .Lavx
 990         test    \$`1<<5`,%ecx
 991         jnz     _avx2_shortcut
 992         jmp     .Lavx
 993 .align  32
 994 .Lavx:
 995 ___
 996 $code.=<<___;
 997         mov     %rsp,%rax
 998         push    %rbx
 999         push    %rbp
1000 ___
1001 $code.=<<___ if ($win64);
1002         lea     -0xa8(%rsp),%rsp
1003         movaps  %xmm6,(%rsp)
1004         movaps  %xmm7,0x10(%rsp)
1005         movaps  %xmm8,0x20(%rsp)
1006         movaps  %xmm9,0x30(%rsp)
1007         movaps  %xmm10,-0x78(%rax)
1008         movaps  %xmm11,-0x68(%rax)
1009         movaps  %xmm12,-0x58(%rax)
1010         movaps  %xmm13,-0x48(%rax)
1011         movaps  %xmm14,-0x38(%rax)
1012         movaps  %xmm15,-0x28(%rax)
1013 ___
1014 $code.=<<___;
1015         sub     \$`$REG_SZ*18`, %rsp
1016         and     \$-256,%rsp
1017         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1018 .Lbody_avx:
1019         lea     K_XX_XX(%rip),$Tbl
1020         lea     `$REG_SZ*16`(%rsp),%rbx
1021
1022         vzeroupper
1023 .Loop_grande_avx:
1024         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1025         xor     $num,$num
1026 ___
1027 for($i=0;$i<4;$i++) {
1028     $code.=<<___;
1029         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1030         mov     `16*$i+8`($inp),%ecx            # number of blocks
1031         cmp     $num,%ecx
1032         cmovg   %ecx,$num                       # find maximum
1033         test    %ecx,%ecx
1034         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1035         cmovle  $Tbl,@ptr[$i]                   # cancel input
1036 ___
1037 }
1038 $code.=<<___;
1039         test    $num,$num
1040         jz      .Ldone_avx
1041
1042         vmovdqu 0x00($ctx),$A                   # load context
1043          lea    128(%rsp),%rax
1044         vmovdqu 0x20($ctx),$B
1045         vmovdqu 0x40($ctx),$C
1046         vmovdqu 0x60($ctx),$D
1047         vmovdqu 0x80($ctx),$E
1048         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1049         jmp     .Loop_avx
1050
1051 .align  32
1052 .Loop_avx:
1053 ___
1054 $code.="        vmovdqa -0x20($Tbl),$K\n";      # K_00_19
1055 for($i=0;$i<20;$i++)    { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1056 $code.="        vmovdqa 0x00($Tbl),$K\n";       # K_20_39
1057 for(;$i<40;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1058 $code.="        vmovdqa 0x20($Tbl),$K\n";       # K_40_59
1059 for(;$i<60;$i++)        { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1060 $code.="        vmovdqa 0x40($Tbl),$K\n";       # K_60_79
1061 for(;$i<80;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1062 $code.=<<___;
1063         mov     \$1,%ecx
1064 ___
1065 for($i=0;$i<4;$i++) {
1066     $code.=<<___;
1067         cmp     `4*$i`(%rbx),%ecx               # examine counters
1068         cmovge  $Tbl,@ptr[$i]                   # cancel input
1069 ___
1070 }
1071 $code.=<<___;
1072         vmovdqu (%rbx),$t0                      # pull counters
1073         vpxor   $t2,$t2,$t2
1074         vmovdqa $t0,$t1
1075         vpcmpgtd $t2,$t1,$t1                    # mask value
1076         vpaddd  $t1,$t0,$t0                     # counters--
1077
1078         vpand   $t1,$A,$A
1079         vpand   $t1,$B,$B
1080         vpaddd  0x00($ctx),$A,$A
1081         vpand   $t1,$C,$C
1082         vpaddd  0x20($ctx),$B,$B
1083         vpand   $t1,$D,$D
1084         vpaddd  0x40($ctx),$C,$C
1085         vpand   $t1,$E,$E
1086         vpaddd  0x60($ctx),$D,$D
1087         vpaddd  0x80($ctx),$E,$E
1088         vmovdqu $A,0x00($ctx)
1089         vmovdqu $B,0x20($ctx)
1090         vmovdqu $C,0x40($ctx)
1091         vmovdqu $D,0x60($ctx)
1092         vmovdqu $E,0x80($ctx)
1093
1094         vmovdqu $t0,(%rbx)                      # save counters
1095         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1096         dec     $num
1097         jnz     .Loop_avx
1098
1099         mov     `$REG_SZ*17+8`(%rsp),$num
1100         lea     $REG_SZ($ctx),$ctx
1101         lea     `16*$REG_SZ/4`($inp),$inp
1102         dec     $num
1103         jnz     .Loop_grande_avx
1104
1105 .Ldone_avx:
1106         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
1107         vzeroupper
1108 ___
1109 $code.=<<___ if ($win64);
1110         movaps  -0xb8(%rax),%xmm6
1111         movaps  -0xa8(%rax),%xmm7
1112         movaps  -0x98(%rax),%xmm8
1113         movaps  -0x88(%rax),%xmm9
1114         movaps  -0x78(%rax),%xmm10
1115         movaps  -0x68(%rax),%xmm11
1116         movaps  -0x58(%rax),%xmm12
1117         movaps  -0x48(%rax),%xmm13
1118         movaps  -0x38(%rax),%xmm14
1119         movaps  -0x28(%rax),%xmm15
1120 ___
1121 $code.=<<___;
1122         mov     -16(%rax),%rbp
1123         mov     -8(%rax),%rbx
1124         lea     (%rax),%rsp
1125 .Lepilogue_avx:
1126         ret
1127 .size   sha1_multi_block_avx,.-sha1_multi_block_avx
1128 ___
1129
1130                                                 if ($avx>1) {
1131 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1132
1133 $REG_SZ=32;
1134
1135 @ptr=map("%r$_",(12..15,8..11));
1136
1137 @V=($A,$B,$C,$D,$E)=map("%ymm$_",(0..4));
1138 ($t0,$t1,$t2,$t3,$tx)=map("%ymm$_",(5..9));
1139 @Xi=map("%ymm$_",(10..14));
1140 $K="%ymm15";
1141
1142 $code.=<<___;
1143 .type   sha1_multi_block_avx2,\@function,3
1144 .align  32
1145 sha1_multi_block_avx2:
1146 _avx2_shortcut:
1147         mov     %rsp,%rax
1148         push    %rbx
1149         push    %rbp
1150         push    %r12
1151         push    %r13
1152         push    %r14
1153         push    %r15
1154 ___
1155 $code.=<<___ if ($win64);
1156         lea     -0xa8(%rsp),%rsp
1157         movaps  %xmm6,(%rsp)
1158         movaps  %xmm7,0x10(%rsp)
1159         movaps  %xmm8,0x20(%rsp)
1160         movaps  %xmm9,0x30(%rsp)
1161         movaps  %xmm10,0x40(%rsp)
1162         movaps  %xmm11,0x50(%rsp)
1163         movaps  %xmm12,-0x78(%rax)
1164         movaps  %xmm13,-0x68(%rax)
1165         movaps  %xmm14,-0x58(%rax)
1166         movaps  %xmm15,-0x48(%rax)
1167 ___
1168 $code.=<<___;
1169         sub     \$`$REG_SZ*18`, %rsp
1170         and     \$-256,%rsp
1171         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1172 .Lbody_avx2:
1173         lea     K_XX_XX(%rip),$Tbl
1174         shr     \$1,$num
1175
1176         vzeroupper
1177 .Loop_grande_avx2:
1178         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1179         xor     $num,$num
1180         lea     `$REG_SZ*16`(%rsp),%rbx
1181 ___
1182 for($i=0;$i<8;$i++) {
1183     $code.=<<___;
1184         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1185         mov     `16*$i+8`($inp),%ecx            # number of blocks
1186         cmp     $num,%ecx
1187         cmovg   %ecx,$num                       # find maximum
1188         test    %ecx,%ecx
1189         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1190         cmovle  $Tbl,@ptr[$i]                   # cancel input
1191 ___
1192 }
1193 $code.=<<___;
1194         vmovdqu 0x00($ctx),$A                   # load context
1195          lea    128(%rsp),%rax
1196         vmovdqu 0x20($ctx),$B
1197          lea    256+128(%rsp),%rbx
1198         vmovdqu 0x40($ctx),$C
1199         vmovdqu 0x60($ctx),$D
1200         vmovdqu 0x80($ctx),$E
1201         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1202         jmp     .Loop_avx2
1203
1204 .align  32
1205 .Loop_avx2:
1206 ___
1207 $code.="        vmovdqa -0x20($Tbl),$K\n";      # K_00_19
1208 for($i=0;$i<20;$i++)    { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1209 $code.="        vmovdqa 0x00($Tbl),$K\n";       # K_20_39
1210 for(;$i<40;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1211 $code.="        vmovdqa 0x20($Tbl),$K\n";       # K_40_59
1212 for(;$i<60;$i++)        { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1213 $code.="        vmovdqa 0x40($Tbl),$K\n";       # K_60_79
1214 for(;$i<80;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1215 $code.=<<___;
1216         mov     \$1,%ecx
1217         lea     `$REG_SZ*16`(%rsp),%rbx
1218 ___
1219 for($i=0;$i<8;$i++) {
1220     $code.=<<___;
1221         cmp     `4*$i`(%rbx),%ecx               # examine counters
1222         cmovge  $Tbl,@ptr[$i]                   # cancel input
1223 ___
1224 }
1225 $code.=<<___;
1226         vmovdqu (%rbx),$t0              # pull counters
1227         vpxor   $t2,$t2,$t2
1228         vmovdqa $t0,$t1
1229         vpcmpgtd $t2,$t1,$t1                    # mask value
1230         vpaddd  $t1,$t0,$t0                     # counters--
1231
1232         vpand   $t1,$A,$A
1233         vpand   $t1,$B,$B
1234         vpaddd  0x00($ctx),$A,$A
1235         vpand   $t1,$C,$C
1236         vpaddd  0x20($ctx),$B,$B
1237         vpand   $t1,$D,$D
1238         vpaddd  0x40($ctx),$C,$C
1239         vpand   $t1,$E,$E
1240         vpaddd  0x60($ctx),$D,$D
1241         vpaddd  0x80($ctx),$E,$E
1242         vmovdqu $A,0x00($ctx)
1243         vmovdqu $B,0x20($ctx)
1244         vmovdqu $C,0x40($ctx)
1245         vmovdqu $D,0x60($ctx)
1246         vmovdqu $E,0x80($ctx)
1247
1248         vmovdqu $t0,(%rbx)                      # save counters
1249         lea     256+128(%rsp),%rbx
1250         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1251         dec     $num
1252         jnz     .Loop_avx2
1253
1254         #mov    `$REG_SZ*17+8`(%rsp),$num
1255         #lea    $REG_SZ($ctx),$ctx
1256         #lea    `16*$REG_SZ/4`($inp),$inp
1257         #dec    $num
1258         #jnz    .Loop_grande_avx2
1259
1260 .Ldone_avx2:
1261         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
1262         vzeroupper
1263 ___
1264 $code.=<<___ if ($win64);
1265         movaps  -0xd8(%rax),%xmm6
1266         movaps  -0xc8(%rax),%xmm7
1267         movaps  -0xb8(%rax),%xmm8
1268         movaps  -0xa8(%rax),%xmm9
1269         movaps  -0x98(%rax),%xmm10
1270         movaps  -0x88(%rax),%xmm11
1271         movaps  -0x78(%rax),%xmm12
1272         movaps  -0x68(%rax),%xmm13
1273         movaps  -0x58(%rax),%xmm14
1274         movaps  -0x48(%rax),%xmm15
1275 ___
1276 $code.=<<___;
1277         mov     -48(%rax),%r15
1278         mov     -40(%rax),%r14
1279         mov     -32(%rax),%r13
1280         mov     -24(%rax),%r12
1281         mov     -16(%rax),%rbp
1282         mov     -8(%rax),%rbx
1283         lea     (%rax),%rsp
1284 .Lepilogue_avx2:
1285         ret
1286 .size   sha1_multi_block_avx2,.-sha1_multi_block_avx2
1287 ___
1288                                                 }       }}}
1289 $code.=<<___;
1290
1291 .align  256
1292         .long   0x5a827999,0x5a827999,0x5a827999,0x5a827999     # K_00_19
1293         .long   0x5a827999,0x5a827999,0x5a827999,0x5a827999     # K_00_19
1294 K_XX_XX:
1295         .long   0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1     # K_20_39
1296         .long   0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1     # K_20_39
1297         .long   0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc     # K_40_59
1298         .long   0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc     # K_40_59
1299         .long   0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6     # K_60_79
1300         .long   0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6     # K_60_79
1301         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1302         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1303         .byte   0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0
1304         .asciz  "SHA1 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
1305 ___
1306
1307 if ($win64) {
1308 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1309 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1310 $rec="%rcx";
1311 $frame="%rdx";
1312 $context="%r8";
1313 $disp="%r9";
1314
1315 $code.=<<___;
1316 .extern __imp_RtlVirtualUnwind
1317 .type   se_handler,\@abi-omnipotent
1318 .align  16
1319 se_handler:
1320         push    %rsi
1321         push    %rdi
1322         push    %rbx
1323         push    %rbp
1324         push    %r12
1325         push    %r13
1326         push    %r14
1327         push    %r15
1328         pushfq
1329         sub     \$64,%rsp
1330
1331         mov     120($context),%rax      # pull context->Rax
1332         mov     248($context),%rbx      # pull context->Rip
1333
1334         mov     8($disp),%rsi           # disp->ImageBase
1335         mov     56($disp),%r11          # disp->HandlerData
1336
1337         mov     0(%r11),%r10d           # HandlerData[0]
1338         lea     (%rsi,%r10),%r10        # end of prologue label
1339         cmp     %r10,%rbx               # context->Rip<.Lbody
1340         jb      .Lin_prologue
1341
1342         mov     152($context),%rax      # pull context->Rsp
1343
1344         mov     4(%r11),%r10d           # HandlerData[1]
1345         lea     (%rsi,%r10),%r10        # epilogue label
1346         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1347         jae     .Lin_prologue
1348
1349         mov     `16*17`(%rax),%rax      # pull saved stack pointer
1350
1351         mov     -8(%rax),%rbx
1352         mov     -16(%rax),%rbp
1353         mov     %rbx,144($context)      # restore context->Rbx
1354         mov     %rbp,160($context)      # restore context->Rbp
1355
1356         lea     -24-10*16(%rax),%rsi
1357         lea     512($context),%rdi      # &context.Xmm6
1358         mov     \$20,%ecx
1359         .long   0xa548f3fc              # cld; rep movsq
1360
1361 .Lin_prologue:
1362         mov     8(%rax),%rdi
1363         mov     16(%rax),%rsi
1364         mov     %rax,152($context)      # restore context->Rsp
1365         mov     %rsi,168($context)      # restore context->Rsi
1366         mov     %rdi,176($context)      # restore context->Rdi
1367
1368         mov     40($disp),%rdi          # disp->ContextRecord
1369         mov     $context,%rsi           # context
1370         mov     \$154,%ecx              # sizeof(CONTEXT)
1371         .long   0xa548f3fc              # cld; rep movsq
1372
1373         mov     $disp,%rsi
1374         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1375         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1376         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1377         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1378         mov     40(%rsi),%r10           # disp->ContextRecord
1379         lea     56(%rsi),%r11           # &disp->HandlerData
1380         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1381         mov     %r10,32(%rsp)           # arg5
1382         mov     %r11,40(%rsp)           # arg6
1383         mov     %r12,48(%rsp)           # arg7
1384         mov     %rcx,56(%rsp)           # arg8, (NULL)
1385         call    *__imp_RtlVirtualUnwind(%rip)
1386
1387         mov     \$1,%eax                # ExceptionContinueSearch
1388         add     \$64,%rsp
1389         popfq
1390         pop     %r15
1391         pop     %r14
1392         pop     %r13
1393         pop     %r12
1394         pop     %rbp
1395         pop     %rbx
1396         pop     %rdi
1397         pop     %rsi
1398         ret
1399 .size   se_handler,.-se_handler
1400 ___
1401 $code.=<<___ if ($avx>1);
1402 .type   avx2_handler,\@abi-omnipotent
1403 .align  16
1404 avx2_handler:
1405         push    %rsi
1406         push    %rdi
1407         push    %rbx
1408         push    %rbp
1409         push    %r12
1410         push    %r13
1411         push    %r14
1412         push    %r15
1413         pushfq
1414         sub     \$64,%rsp
1415
1416         mov     120($context),%rax      # pull context->Rax
1417         mov     248($context),%rbx      # pull context->Rip
1418
1419         mov     8($disp),%rsi           # disp->ImageBase
1420         mov     56($disp),%r11          # disp->HandlerData
1421
1422         mov     0(%r11),%r10d           # HandlerData[0]
1423         lea     (%rsi,%r10),%r10        # end of prologue label
1424         cmp     %r10,%rbx               # context->Rip<body label
1425         jb      .Lin_prologue
1426
1427         mov     152($context),%rax      # pull context->Rsp
1428
1429         mov     4(%r11),%r10d           # HandlerData[1]
1430         lea     (%rsi,%r10),%r10        # epilogue label
1431         cmp     %r10,%rbx               # context->Rip>=epilogue label
1432         jae     .Lin_prologue
1433
1434         mov     `32*17`($context),%rax  # pull saved stack pointer
1435
1436         mov     -8(%rax),%rbx
1437         mov     -16(%rax),%rbp
1438         mov     -24(%rax),%r12
1439         mov     -32(%rax),%r13
1440         mov     -40(%rax),%r14
1441         mov     -48(%rax),%r15
1442         mov     %rbx,144($context)      # restore context->Rbx
1443         mov     %rbp,160($context)      # restore context->Rbp
1444         mov     %r12,216($context)      # restore cotnext->R12
1445         mov     %r13,224($context)      # restore cotnext->R13
1446         mov     %r14,232($context)      # restore cotnext->R14
1447         mov     %r15,240($context)      # restore cotnext->R15
1448
1449         lea     -56-10*16(%rax),%rsi
1450         lea     512($context),%rdi      # &context.Xmm6
1451         mov     \$20,%ecx
1452         .long   0xa548f3fc              # cld; rep movsq
1453
1454         jmp     .Lin_prologue
1455 .size   avx2_handler,.-avx2_handler
1456 ___
1457 $code.=<<___;
1458 .section        .pdata
1459 .align  4
1460         .rva    .LSEH_begin_sha1_multi_block
1461         .rva    .LSEH_end_sha1_multi_block
1462         .rva    .LSEH_info_sha1_multi_block
1463         .rva    .LSEH_begin_sha1_multi_block_shaext
1464         .rva    .LSEH_end_sha1_multi_block_shaext
1465         .rva    .LSEH_info_sha1_multi_block_shaext
1466 ___
1467 $code.=<<___ if ($avx);
1468         .rva    .LSEH_begin_sha1_multi_block_avx
1469         .rva    .LSEH_end_sha1_multi_block_avx
1470         .rva    .LSEH_info_sha1_multi_block_avx
1471 ___
1472 $code.=<<___ if ($avx>1);
1473         .rva    .LSEH_begin_sha1_multi_block_avx2
1474         .rva    .LSEH_end_sha1_multi_block_avx2
1475         .rva    .LSEH_info_sha1_multi_block_avx2
1476 ___
1477 $code.=<<___;
1478 .section        .xdata
1479 .align  8
1480 .LSEH_info_sha1_multi_block:
1481         .byte   9,0,0,0
1482         .rva    se_handler
1483         .rva    .Lbody,.Lepilogue                       # HandlerData[]
1484 .LSEH_info_sha1_multi_block_shaext:
1485         .byte   9,0,0,0
1486         .rva    se_handler
1487         .rva    .Lbody_shaext,.Lepilogue_shaext # HandlerData[]
1488 ___
1489 $code.=<<___ if ($avx);
1490 .LSEH_info_sha1_multi_block_avx:
1491         .byte   9,0,0,0
1492         .rva    se_handler
1493         .rva    .Lbody_avx,.Lepilogue_avx               # HandlerData[]
1494 ___
1495 $code.=<<___ if ($avx>1);
1496 .LSEH_info_sha1_multi_block_avx2:
1497         .byte   9,0,0,0
1498         .rva    avx2_handler
1499         .rva    .Lbody_avx2,.Lepilogue_avx2             # HandlerData[]
1500 ___
1501 }
1502 ####################################################################
1503
1504 sub rex {
1505   local *opcode=shift;
1506   my ($dst,$src)=@_;
1507   my $rex=0;
1508
1509     $rex|=0x04                  if ($dst>=8);
1510     $rex|=0x01                  if ($src>=8);
1511     unshift @opcode,$rex|0x40   if ($rex);
1512 }
1513
1514 sub sha1rnds4 {
1515     if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1516       my @opcode=(0x0f,0x3a,0xcc);
1517         rex(\@opcode,$3,$2);
1518         push @opcode,0xc0|($2&7)|(($3&7)<<3);           # ModR/M
1519         my $c=$1;
1520         push @opcode,$c=~/^0/?oct($c):$c;
1521         return ".byte\t".join(',',@opcode);
1522     } else {
1523         return "sha1rnds4\t".@_[0];
1524     }
1525 }
1526
1527 sub sha1op38 {
1528     my $instr = shift;
1529     my %opcodelet = (
1530                 "sha1nexte" => 0xc8,
1531                 "sha1msg1"  => 0xc9,
1532                 "sha1msg2"  => 0xca     );
1533
1534     if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1535       my @opcode=(0x0f,0x38);
1536         rex(\@opcode,$2,$1);
1537         push @opcode,$opcodelet{$instr};
1538         push @opcode,0xc0|($1&7)|(($2&7)<<3);           # ModR/M
1539         return ".byte\t".join(',',@opcode);
1540     } else {
1541         return $instr."\t".@_[0];
1542     }
1543 }
1544
1545 foreach (split("\n",$code)) {
1546         s/\`([^\`]*)\`/eval($1)/ge;
1547
1548         s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo                or
1549         s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo             or
1550
1551         s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go          or
1552         s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go         or
1553         s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go    or
1554         s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go        or
1555         s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go            or
1556         s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;
1557
1558         print $_,"\n";
1559 }
1560
1561 close STDOUT;