crypto/sha/asm/sha1-mb-x86_64.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9
  10 # Multi-buffer SHA1 procedure processes n buffers in parallel by
  11 # placing buffer data to designated lane of SIMD register. n is
  12 # naturally limited to 4 on pre-AVX2 processors and to 8 on
  13 # AVX2-capable processors such as Haswell.
  14 #
  15 #               this    +aesni(i)       sha1    aesni-sha1      gain(iv)
  16 # -------------------------------------------------------------------
  17 # Westmere(ii)  10.7/n  +1.28=3.96(n=4) 5.30    6.66            +68%
  18 # Atom(ii)      18.1/n  +3.93=8.46(n=4) 9.37    12.8            +51%
  19 # Sandy Bridge  (8.16   +5.15=13.3)/n   4.99    5.98            +80%
  20 # Ivy Bridge    (8.08   +5.14=13.2)/n   4.60    5.54            +68%
  21 # Haswell(iii)  (8.96   +5.00=14.0)/n   3.57    4.55            +160%
  22 # Bulldozer     (9.76   +5.76=15.5)/n   5.95    6.37            +64%
  23 #
  24 # (i)   multi-block CBC encrypt with 128-bit key;
  25 # (ii)  (HASH+AES)/n does not apply to Westmere for n>3 and Atom,
  26 #       because of lower AES-NI instruction throughput;
  27 # (iii) "this" is for n=8, when we gather twice as much data, result
  28 #       for n=4 is 8.00+4.44=12.4;
  29 # (iv)  presented improvement coefficients are asymptotic limits and
  30 #       in real-life application are somewhat lower, e.g. for 2KB
  31 #       fragments they range from 30% to 100% (on Haswell);
  32
  33 $flavour = shift;
  34 $output  = shift;
  35 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  36
  37 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  38
  39 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  40 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  41 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  42 die "can't locate x86_64-xlate.pl";
  43
  44 $avx=0;
  45
  46 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  47                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  48         $avx = ($1>=2.19) + ($1>=2.22);
  49 }
  50
  51 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  52            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  53         $avx = ($1>=2.09) + ($1>=2.10);
  54 }
  55
  56 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  57            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  58         $avx = ($1>=10) + ($1>=11);
  59 }
  60
  61 open OUT,"| \"$^X\" $xlate $flavour $output";
  62 *STDOUT=*OUT;
  63
  64 # void sha1_multi_block (
  65 #     struct {  unsigned int A[8];
  66 #               unsigned int B[8];
  67 #               unsigned int C[8];
  68 #               unsigned int D[8];
  69 #               unsigned int E[8];      } *ctx,
  70 #     struct {  void *ptr; int blocks;  } inp[8],
  71 #     int num);         /* 1 or 2 */
  72 #
  73 $ctx="%rdi";    # 1st arg
  74 $inp="%rsi";    # 2nd arg
  75 $num="%edx";
  76 @ptr=map("%r$_",(8..11));
  77 $Tbl="%rbp";
  78
  79 @V=($A,$B,$C,$D,$E)=map("%xmm$_",(0..4));
  80 ($t0,$t1,$t2,$t3,$tx)=map("%xmm$_",(5..9));
  81 @Xi=map("%xmm$_",(10..14));
  82 $K="%xmm15";
  83
  84 if (1) {
  85     # Atom-specific optimization aiming to eliminate pshufb with high
  86     # registers [and thus get rid of 48 cycles accumulated penalty]
  87     @Xi=map("%xmm$_",(0..4));
  88     ($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9));
  89     @V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14));
  90 }
  91
  92 $REG_SZ=16;
  93
  94 sub Xi_off {
  95 my $off = shift;
  96
  97     $off %= 16; $off *= $REG_SZ;
  98     $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
  99 }
 100
 101 sub BODY_00_19 {
 102 my ($i,$a,$b,$c,$d,$e)=@_;
 103 my $j=$i+1;
 104 my $k=$i+2;
 105
 106 $code.=<<___ if ($i==0);
 107         movd            (@ptr[0]),@Xi[0]
 108          lea            `16*4`(@ptr[0]),@ptr[0]
 109         movd            (@ptr[1]),@Xi[2]        # borrow @Xi[2]
 110          lea            `16*4`(@ptr[1]),@ptr[1]
 111         movd            (@ptr[2]),@Xi[3]        # borrow @Xi[3]
 112          lea            `16*4`(@ptr[2]),@ptr[2]
 113         movd            (@ptr[3]),@Xi[4]        # borrow @Xi[4]
 114          lea            `16*4`(@ptr[3]),@ptr[3]
 115         punpckldq       @Xi[3],@Xi[0]
 116          movd           `4*$j-16*4`(@ptr[0]),@Xi[1]
 117         punpckldq       @Xi[4],@Xi[2]
 118          movd           `4*$j-16*4`(@ptr[1]),$t3
 119         punpckldq       @Xi[2],@Xi[0]
 120          movd           `4*$j-16*4`(@ptr[2]),$t2
 121         pshufb          $tx,@Xi[0]
 122 ___
 123 $code.=<<___ if ($i<14);                        # just load input
 124          movd           `4*$j-16*4`(@ptr[3]),$t1
 125          punpckldq      $t2,@Xi[1]
 126         movdqa  $a,$t2
 127         paddd   $K,$e                           # e+=K_00_19
 128          punpckldq      $t1,$t3
 129         movdqa  $b,$t1
 130         movdqa  $b,$t0
 131         pslld   \$5,$t2
 132         pandn   $d,$t1
 133         pand    $c,$t0
 134          punpckldq      $t3,@Xi[1]
 135         movdqa  $a,$t3
 136
 137         movdqa  @Xi[0],`&Xi_off($i)`
 138         paddd   @Xi[0],$e                       # e+=X[i]
 139          movd           `4*$k-16*4`(@ptr[0]),@Xi[2]
 140         psrld   \$27,$t3
 141         pxor    $t1,$t0                         # Ch(b,c,d)
 142         movdqa  $b,$t1
 143
 144         por     $t3,$t2                         # rol(a,5)
 145          movd           `4*$k-16*4`(@ptr[1]),$t3
 146         pslld   \$30,$t1
 147         paddd   $t0,$e                          # e+=Ch(b,c,d)
 148
 149         psrld   \$2,$b
 150         paddd   $t2,$e                          # e+=rol(a,5)
 151          pshufb $tx,@Xi[1]
 152          movd           `4*$j-16*4`(@ptr[2]),$t2
 153         por     $t1,$b                          # b=rol(b,30)
 154 ___
 155 $code.=<<___ if ($i==14);                       # just load input
 156          movd           `4*$j-16*4`(@ptr[3]),$t1
 157          punpckldq      $t2,@Xi[1]
 158         movdqa  $a,$t2
 159         paddd   $K,$e                           # e+=K_00_19
 160          punpckldq      $t1,$t3
 161         movdqa  $b,$t1
 162         movdqa  $b,$t0
 163         pslld   \$5,$t2
 164          prefetcht0     63(@ptr[0])
 165         pandn   $d,$t1
 166         pand    $c,$t0
 167          punpckldq      $t3,@Xi[1]
 168         movdqa  $a,$t3
 169
 170         movdqa  @Xi[0],`&Xi_off($i)`
 171         paddd   @Xi[0],$e                       # e+=X[i]
 172         psrld   \$27,$t3
 173         pxor    $t1,$t0                         # Ch(b,c,d)
 174         movdqa  $b,$t1
 175          prefetcht0     63(@ptr[1])
 176
 177         por     $t3,$t2                         # rol(a,5)
 178         pslld   \$30,$t1
 179         paddd   $t0,$e                          # e+=Ch(b,c,d)
 180          prefetcht0     63(@ptr[2])
 181
 182         psrld   \$2,$b
 183         paddd   $t2,$e                          # e+=rol(a,5)
 184          pshufb $tx,@Xi[1]
 185          prefetcht0     63(@ptr[3])
 186         por     $t1,$b                          # b=rol(b,30)
 187 ___
 188 $code.=<<___ if ($i>=13 && $i<15);
 189         movdqa  `&Xi_off($j+2)`,@Xi[3]          # preload "X[2]"
 190 ___
 191 $code.=<<___ if ($i>=15);                       # apply Xupdate
 192         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 193         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 194
 195         movdqa  $a,$t2
 196          pxor   `&Xi_off($j+8)`,@Xi[1]
 197         paddd   $K,$e                           # e+=K_00_19
 198         movdqa  $b,$t1
 199         pslld   \$5,$t2
 200          pxor   @Xi[3],@Xi[1]
 201         movdqa  $b,$t0
 202         pandn   $d,$t1
 203          movdqa @Xi[1],$tx
 204         pand    $c,$t0
 205         movdqa  $a,$t3
 206          psrld  \$31,$tx
 207          paddd  @Xi[1],@Xi[1]
 208
 209         movdqa  @Xi[0],`&Xi_off($i)`
 210         paddd   @Xi[0],$e                       # e+=X[i]
 211         psrld   \$27,$t3
 212         pxor    $t1,$t0                         # Ch(b,c,d)
 213
 214         movdqa  $b,$t1
 215         por     $t3,$t2                         # rol(a,5)
 216         pslld   \$30,$t1
 217         paddd   $t0,$e                          # e+=Ch(b,c,d)
 218
 219         psrld   \$2,$b
 220         paddd   $t2,$e                          # e+=rol(a,5)
 221          por    $tx,@Xi[1]                      # rol   \$1,@Xi[1]
 222         por     $t1,$b                          # b=rol(b,30)
 223 ___
 224 push(@Xi,shift(@Xi));
 225 }
 226
 227 sub BODY_20_39 {
 228 my ($i,$a,$b,$c,$d,$e)=@_;
 229 my $j=$i+1;
 230
 231 $code.=<<___ if ($i<79);
 232         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 233         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 234
 235         movdqa  $a,$t2
 236         movdqa  $d,$t0
 237          pxor   `&Xi_off($j+8)`,@Xi[1]
 238         paddd   $K,$e                           # e+=K_20_39
 239         pslld   \$5,$t2
 240         pxor    $b,$t0
 241
 242         movdqa  $a,$t3
 243 ___
 244 $code.=<<___ if ($i<72);
 245         movdqa  @Xi[0],`&Xi_off($i)`
 246 ___
 247 $code.=<<___ if ($i<79);
 248         paddd   @Xi[0],$e                       # e+=X[i]
 249          pxor   @Xi[3],@Xi[1]
 250         psrld   \$27,$t3
 251         pxor    $c,$t0                          # Parity(b,c,d)
 252         movdqa  $b,$t1
 253
 254         pslld   \$30,$t1
 255          movdqa @Xi[1],$tx
 256         por     $t3,$t2                         # rol(a,5)
 257          psrld  \$31,$tx
 258         paddd   $t0,$e                          # e+=Parity(b,c,d)
 259          paddd  @Xi[1],@Xi[1]
 260
 261         psrld   \$2,$b
 262         paddd   $t2,$e                          # e+=rol(a,5)
 263          por    $tx,@Xi[1]                      # rol(@Xi[1],1)
 264         por     $t1,$b                          # b=rol(b,30)
 265 ___
 266 $code.=<<___ if ($i==79);
 267         movdqa  $a,$t2
 268         paddd   $K,$e                           # e+=K_20_39
 269         movdqa  $d,$t0
 270         pslld   \$5,$t2
 271         pxor    $b,$t0
 272
 273         movdqa  $a,$t3
 274         paddd   @Xi[0],$e                       # e+=X[i]
 275         psrld   \$27,$t3
 276         movdqa  $b,$t1
 277         pxor    $c,$t0                          # Parity(b,c,d)
 278
 279         pslld   \$30,$t1
 280         por     $t3,$t2                         # rol(a,5)
 281         paddd   $t0,$e                          # e+=Parity(b,c,d)
 282
 283         psrld   \$2,$b
 284         paddd   $t2,$e                          # e+=rol(a,5)
 285         por     $t1,$b                          # b=rol(b,30)
 286 ___
 287 push(@Xi,shift(@Xi));
 288 }
 289
 290 sub BODY_40_59 {
 291 my ($i,$a,$b,$c,$d,$e)=@_;
 292 my $j=$i+1;
 293
 294 $code.=<<___;
 295         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 296         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 297
 298         movdqa  $a,$t2
 299         movdqa  $d,$t1
 300          pxor   `&Xi_off($j+8)`,@Xi[1]
 301         pxor    @Xi[3],@Xi[1]
 302         paddd   $K,$e                           # e+=K_40_59
 303         pslld   \$5,$t2
 304         movdqa  $a,$t3
 305         pand    $c,$t1
 306
 307         movdqa  $d,$t0
 308          movdqa @Xi[1],$tx
 309         psrld   \$27,$t3
 310         paddd   $t1,$e
 311         pxor    $c,$t0
 312
 313         movdqa  @Xi[0],`&Xi_off($i)`
 314         paddd   @Xi[0],$e                       # e+=X[i]
 315         por     $t3,$t2                         # rol(a,5)
 316          psrld  \$31,$tx
 317         pand    $b,$t0
 318         movdqa  $b,$t1
 319
 320         pslld   \$30,$t1
 321          paddd  @Xi[1],@Xi[1]
 322         paddd   $t0,$e                          # e+=Maj(b,d,c)
 323
 324         psrld   \$2,$b
 325         paddd   $t2,$e                          # e+=rol(a,5)
 326          por    $tx,@Xi[1]                      # rol(@X[1],1)
 327         por     $t1,$b                          # b=rol(b,30)
 328 ___
 329 push(@Xi,shift(@Xi));
 330 }
 331
 332 $code.=<<___;
 333 .text
 334
 335 .extern OPENSSL_ia32cap_P
 336
 337 .globl  sha1_multi_block
 338 .type   sha1_multi_block,\@function,3
 339 .align  32
 340 sha1_multi_block:
 341         mov     OPENSSL_ia32cap_P+4(%rip),%rcx
 342         bt      \$61,%rcx                       # check SHA bit
 343         jc      _shaext_shortcut
 344 ___
 345 $code.=<<___ if ($avx);
 346         test    \$`1<<28`,%ecx
 347         jnz     _avx_shortcut
 348 ___
 349 $code.=<<___;
 350         mov     %rsp,%rax
 351         push    %rbx
 352         push    %rbp
 353 ___
 354 $code.=<<___ if ($win64);
 355         lea     -0xa8(%rsp),%rsp
 356         movaps  %xmm6,(%rsp)
 357         movaps  %xmm7,0x10(%rsp)
 358         movaps  %xmm8,0x20(%rsp)
 359         movaps  %xmm9,0x30(%rsp)
 360         movaps  %xmm10,-0x78(%rax)
 361         movaps  %xmm11,-0x68(%rax)
 362         movaps  %xmm12,-0x58(%rax)
 363         movaps  %xmm13,-0x48(%rax)
 364         movaps  %xmm14,-0x38(%rax)
 365         movaps  %xmm15,-0x28(%rax)
 366 ___
 367 $code.=<<___;
 368         sub     \$`$REG_SZ*18`,%rsp
 369         and     \$-256,%rsp
 370         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 371 .Lbody:
 372         lea     K_XX_XX(%rip),$Tbl
 373         lea     `$REG_SZ*16`(%rsp),%rbx
 374
 375 .Loop_grande:
 376         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
 377         xor     $num,$num
 378 ___
 379 for($i=0;$i<4;$i++) {
 380     $code.=<<___;
 381         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 382         mov     `16*$i+8`($inp),%ecx            # number of blocks
 383         cmp     $num,%ecx
 384         cmovg   %ecx,$num                       # find maximum
 385         test    %ecx,%ecx
 386         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 387         cmovle  $Tbl,@ptr[$i]                   # cancel input
 388 ___
 389 }
 390 $code.=<<___;
 391         test    $num,$num
 392         jz      .Ldone
 393
 394         movdqu  0x00($ctx),$A                   # load context
 395          lea    128(%rsp),%rax
 396         movdqu  0x20($ctx),$B
 397         movdqu  0x40($ctx),$C
 398         movdqu  0x60($ctx),$D
 399         movdqu  0x80($ctx),$E
 400         movdqa  0x60($Tbl),$tx                  # pbswap_mask
 401         movdqa  -0x20($Tbl),$K                  # K_00_19
 402         jmp     .Loop
 403
 404 .align  32
 405 .Loop:
 406 ___
 407 for($i=0;$i<20;$i++)    { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
 408 $code.="        movdqa  0x00($Tbl),$K\n";       # K_20_39
 409 for(;$i<40;$i++)        { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
 410 $code.="        movdqa  0x20($Tbl),$K\n";       # K_40_59
 411 for(;$i<60;$i++)        { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
 412 $code.="        movdqa  0x40($Tbl),$K\n";       # K_60_79
 413 for(;$i<80;$i++)        { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
 414 $code.=<<___;
 415         movdqa  (%rbx),@Xi[0]                   # pull counters
 416         mov     \$1,%ecx
 417         cmp     4*0(%rbx),%ecx                  # examinte counters
 418         pxor    $t2,$t2
 419         cmovge  $Tbl,@ptr[0]                    # cancel input
 420         cmp     4*1(%rbx),%ecx
 421         movdqa  @Xi[0],@Xi[1]
 422         cmovge  $Tbl,@ptr[1]
 423         cmp     4*2(%rbx),%ecx
 424         pcmpgtd $t2,@Xi[1]                      # mask value
 425         cmovge  $Tbl,@ptr[2]
 426         cmp     4*3(%rbx),%ecx
 427         paddd   @Xi[1],@Xi[0]                   # counters--
 428         cmovge  $Tbl,@ptr[3]
 429
 430         movdqu  0x00($ctx),$t0
 431         pand    @Xi[1],$A
 432         movdqu  0x20($ctx),$t1
 433         pand    @Xi[1],$B
 434         paddd   $t0,$A
 435         movdqu  0x40($ctx),$t2
 436         pand    @Xi[1],$C
 437         paddd   $t1,$B
 438         movdqu  0x60($ctx),$t3
 439         pand    @Xi[1],$D
 440         paddd   $t2,$C
 441         movdqu  0x80($ctx),$tx
 442         pand    @Xi[1],$E
 443         movdqu  $A,0x00($ctx)
 444         paddd   $t3,$D
 445         movdqu  $B,0x20($ctx)
 446         paddd   $tx,$E
 447         movdqu  $C,0x40($ctx)
 448         movdqu  $D,0x60($ctx)
 449         movdqu  $E,0x80($ctx)
 450
 451         movdqa  @Xi[0],(%rbx)                   # save counters
 452         movdqa  0x60($Tbl),$tx                  # pbswap_mask
 453         movdqa  -0x20($Tbl),$K                  # K_00_19
 454         dec     $num
 455         jnz     .Loop
 456
 457         mov     `$REG_SZ*17+8`(%rsp),$num
 458         lea     $REG_SZ($ctx),$ctx
 459         lea     `16*$REG_SZ/4`($inp),$inp
 460         dec     $num
 461         jnz     .Loop_grande
 462
 463 .Ldone:
 464         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
 465 ___
 466 $code.=<<___ if ($win64);
 467         movaps  -0xb8(%rax),%xmm6
 468         movaps  -0xa8(%rax),%xmm7
 469         movaps  -0x98(%rax),%xmm8
 470         movaps  -0x88(%rax),%xmm9
 471         movaps  -0x78(%rax),%xmm10
 472         movaps  -0x68(%rax),%xmm11
 473         movaps  -0x58(%rax),%xmm12
 474         movaps  -0x48(%rax),%xmm13
 475         movaps  -0x38(%rax),%xmm14
 476         movaps  -0x28(%rax),%xmm15
 477 ___
 478 $code.=<<___;
 479         mov     -16(%rax),%rbp
 480         mov     -8(%rax),%rbx
 481         lea     (%rax),%rsp
 482 .Lepilogue:
 483         ret
 484 .size   sha1_multi_block,.-sha1_multi_block
 485 ___
 486                                                 {{{
 487 my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10));
 488 my @MSG0=map("%xmm$_",(4..7));
 489 my @MSG1=map("%xmm$_",(11..14));
 490
 491 $code.=<<___;
 492 .type   sha1_multi_block_shaext,\@function,3
 493 .align  32
 494 sha1_multi_block_shaext:
 495 _shaext_shortcut:
 496         mov     %rsp,%rax
 497         push    %rbx
 498         push    %rbp
 499 ___
 500 $code.=<<___ if ($win64);
 501         lea     -0xa8(%rsp),%rsp
 502         movaps  %xmm6,(%rsp)
 503         movaps  %xmm7,0x10(%rsp)
 504         movaps  %xmm8,0x20(%rsp)
 505         movaps  %xmm9,0x30(%rsp)
 506         movaps  %xmm10,-0x78(%rax)
 507         movaps  %xmm11,-0x68(%rax)
 508         movaps  %xmm12,-0x58(%rax)
 509         movaps  %xmm13,-0x48(%rax)
 510         movaps  %xmm14,-0x38(%rax)
 511         movaps  %xmm15,-0x28(%rax)
 512 ___
 513 $code.=<<___;
 514         sub     \$`$REG_SZ*18`,%rsp
 515         shl     \$1,$num                        # we process pair at a time
 516         and     \$-256,%rsp
 517         lea     0x40($ctx),$ctx                 # size optimization
 518         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 519 .Lbody_shaext:
 520         lea     `$REG_SZ*16`(%rsp),%rbx
 521         movdqa  K_XX_XX+0x80(%rip),$BSWAP       # byte-n-word swap
 522
 523 .Loop_grande_shaext:
 524         mov     $num,`$REG_SZ*17+8`(%rsp)       # orignal $num
 525         xor     $num,$num
 526 ___
 527 for($i=0;$i<2;$i++) {
 528     $code.=<<___;
 529         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 530         mov     `16*$i+8`($inp),%ecx            # number of blocks
 531         cmp     $num,%ecx
 532         cmovg   %ecx,$num                       # find maximum
 533         test    %ecx,%ecx
 534         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 535         cmovle  %rsp,@ptr[$i]                   # cancel input
 536 ___
 537 }
 538 $code.=<<___;
 539         test    $num,$num
 540         jz      .Ldone_shaext
 541
 542         movq            0x00-0x40($ctx),$ABCD0  # a1.a0
 543         movq            0x20-0x40($ctx),@MSG0[0]# b1.b0
 544         movq            0x40-0x40($ctx),@MSG0[1]# c1.c0
 545         movq            0x60-0x40($ctx),@MSG0[2]# d1.d0
 546         movq            0x80-0x40($ctx),@MSG0[3]# e1.e0
 547
 548         punpckldq       @MSG0[0],$ABCD0         # b1.a1.b0.a0
 549         punpckldq       @MSG0[2],@MSG0[1]       # d1.c1.d0.c0
 550
 551         movdqa          $ABCD0,$ABCD1
 552         punpcklqdq      @MSG0[1],$ABCD0         # d0.c0.b0.a0
 553         punpckhqdq      @MSG0[1],$ABCD1         # d1.c1.b1.a1
 554
 555         pshufd          \$0b00111111,@MSG0[3],$E0
 556         pshufd          \$0b01111111,@MSG0[3],$E1
 557         pshufd          \$0b00011011,$ABCD0,$ABCD0
 558         pshufd          \$0b00011011,$ABCD1,$ABCD1
 559         jmp             .Loop_shaext
 560
 561 .align  32
 562 .Loop_shaext:
 563         movdqu          0x00(@ptr[0]),@MSG0[0]
 564          movdqu         0x00(@ptr[1]),@MSG1[0]
 565         movdqu          0x10(@ptr[0]),@MSG0[1]
 566          movdqu         0x10(@ptr[1]),@MSG1[1]
 567         movdqu          0x20(@ptr[0]),@MSG0[2]
 568         pshufb          $BSWAP,@MSG0[0]
 569          movdqu         0x20(@ptr[1]),@MSG1[2]
 570          pshufb         $BSWAP,@MSG1[0]
 571         movdqu          0x30(@ptr[0]),@MSG0[3]
 572         lea             0x40(@ptr[0]),@ptr[0]
 573         pshufb          $BSWAP,@MSG0[1]
 574          movdqu         0x30(@ptr[1]),@MSG1[3]
 575          lea            0x40(@ptr[1]),@ptr[1]
 576          pshufb         $BSWAP,@MSG1[1]
 577
 578         movdqa          $E0,0x50(%rsp)          # offload
 579         paddd           @MSG0[0],$E0
 580          movdqa         $E1,0x70(%rsp)
 581          paddd          @MSG1[0],$E1
 582         movdqa          $ABCD0,0x40(%rsp)       # offload
 583         movdqa          $ABCD0,$E0_
 584          movdqa         $ABCD1,0x60(%rsp)
 585          movdqa         $ABCD1,$E1_
 586         sha1rnds4       \$0,$E0,$ABCD0          # 0-3
 587         sha1nexte       @MSG0[1],$E0_
 588          sha1rnds4      \$0,$E1,$ABCD1          # 0-3
 589          sha1nexte      @MSG1[1],$E1_
 590         pshufb          $BSWAP,@MSG0[2]
 591         prefetcht0      127(@ptr[0])
 592         sha1msg1        @MSG0[1],@MSG0[0]
 593          pshufb         $BSWAP,@MSG1[2]
 594          prefetcht0     127(@ptr[1])
 595          sha1msg1       @MSG1[1],@MSG1[0]
 596
 597         pshufb          $BSWAP,@MSG0[3]
 598         movdqa          $ABCD0,$E0
 599          pshufb         $BSWAP,@MSG1[3]
 600          movdqa         $ABCD1,$E1
 601         sha1rnds4       \$0,$E0_,$ABCD0         # 4-7
 602         sha1nexte       @MSG0[2],$E0
 603          sha1rnds4      \$0,$E1_,$ABCD1         # 4-7
 604          sha1nexte      @MSG1[2],$E1
 605         pxor            @MSG0[2],@MSG0[0]
 606         sha1msg1        @MSG0[2],@MSG0[1]
 607          pxor           @MSG1[2],@MSG1[0]
 608          sha1msg1       @MSG1[2],@MSG1[1]
 609 ___
 610 for($i=2;$i<20-4;$i++) {
 611 $code.=<<___;
 612         movdqa          $ABCD0,$E0_
 613          movdqa         $ABCD1,$E1_
 614         sha1rnds4       \$`int($i/5)`,$E0,$ABCD0        # 8-11
 615         sha1nexte       @MSG0[3],$E0_
 616          sha1rnds4      \$`int($i/5)`,$E1,$ABCD1        # 8-11
 617          sha1nexte      @MSG1[3],$E1_
 618         sha1msg2        @MSG0[3],@MSG0[0]
 619          sha1msg2       @MSG1[3],@MSG1[0]
 620         pxor            @MSG0[3],@MSG0[1]
 621         sha1msg1        @MSG0[3],@MSG0[2]
 622          pxor           @MSG1[3],@MSG1[1]
 623          sha1msg1       @MSG1[3],@MSG1[2]
 624 ___
 625         ($E0,$E0_)=($E0_,$E0);          ($E1,$E1_)=($E1_,$E1);
 626         push(@MSG0,shift(@MSG0));       push(@MSG1,shift(@MSG1));
 627 }
 628 $code.=<<___;
 629         movdqa          $ABCD0,$E0_
 630          movdqa         $ABCD1,$E1_
 631         sha1rnds4       \$3,$E0,$ABCD0          # 64-67
 632         sha1nexte       @MSG0[3],$E0_
 633          sha1rnds4      \$3,$E1,$ABCD1          # 64-67
 634          sha1nexte      @MSG1[3],$E1_
 635         sha1msg2        @MSG0[3],@MSG0[0]
 636          sha1msg2       @MSG1[3],@MSG1[0]
 637         pxor            @MSG0[3],@MSG0[1]
 638          pxor           @MSG1[3],@MSG1[1]
 639
 640         mov             \$1,%ecx
 641         pxor            @MSG0[2],@MSG0[2]       # zero
 642         cmp             4*0(%rbx),%ecx          # examine counters
 643         cmovge          %rsp,@ptr[0]            # cancel input
 644
 645         movdqa          $ABCD0,$E0
 646          movdqa         $ABCD1,$E1
 647         sha1rnds4       \$3,$E0_,$ABCD0         # 68-71
 648         sha1nexte       @MSG0[0],$E0
 649          sha1rnds4      \$3,$E1_,$ABCD1         # 68-71
 650          sha1nexte      @MSG1[0],$E1
 651         sha1msg2        @MSG0[0],@MSG0[1]
 652          sha1msg2       @MSG1[0],@MSG1[1]
 653
 654         cmp             4*1(%rbx),%ecx
 655         cmovge          %rsp,@ptr[1]
 656         movq            (%rbx),@MSG0[0]         # pull counters
 657
 658         movdqa          $ABCD0,$E0_
 659          movdqa         $ABCD1,$E1_
 660         sha1rnds4       \$3,$E0,$ABCD0          # 72-75
 661         sha1nexte       @MSG0[1],$E0_
 662          sha1rnds4      \$3,$E1,$ABCD1          # 72-75
 663          sha1nexte      @MSG1[1],$E1_
 664
 665         pshufd          \$0x00,@MSG0[0],@MSG1[2]
 666         pshufd          \$0x55,@MSG0[0],@MSG1[3]
 667         movdqa          @MSG0[0],@MSG0[1]
 668         pcmpgtd         @MSG0[2],@MSG1[2]
 669         pcmpgtd         @MSG0[2],@MSG1[3]
 670
 671         movdqa          $ABCD0,$E0
 672          movdqa         $ABCD1,$E1
 673         sha1rnds4       \$3,$E0_,$ABCD0         # 76-79
 674         sha1nexte       $MSG0[2],$E0
 675          sha1rnds4      \$3,$E1_,$ABCD1         # 76-79
 676          sha1nexte      $MSG0[2],$E1
 677
 678         pcmpgtd         @MSG0[2],@MSG0[1]       # counter mask
 679         pand            @MSG1[2],$ABCD0
 680         pand            @MSG1[2],$E0
 681          pand           @MSG1[3],$ABCD1
 682          pand           @MSG1[3],$E1
 683         paddd           @MSG0[1],@MSG0[0]       # counters--
 684
 685         paddd           0x40(%rsp),$ABCD0
 686         paddd           0x50(%rsp),$E0
 687          paddd          0x60(%rsp),$ABCD1
 688          paddd          0x70(%rsp),$E1
 689
 690         movq            @MSG0[0],(%rbx)         # save counters
 691         dec             $num
 692         jnz             .Loop_shaext
 693
 694         mov             `$REG_SZ*17+8`(%rsp),$num
 695
 696         pshufd          \$0b00011011,$ABCD0,$ABCD0
 697         pshufd          \$0b00011011,$ABCD1,$ABCD1
 698
 699         movdqa          $ABCD0,@MSG0[0]
 700         punpckldq       $ABCD1,$ABCD0           # b1.b0.a1.a0
 701         punpckhdq       $ABCD1,@MSG0[0]         # d1.d0.c1.c0
 702         punpckhdq       $E1,$E0                 # e1.e0.xx.xx
 703         movq            $ABCD0,0x00-0x40($ctx)  # a1.a0
 704         psrldq          \$8,$ABCD0
 705         movq            @MSG0[0],0x40-0x40($ctx)# c1.c0
 706         psrldq          \$8,@MSG0[0]
 707         movq            $ABCD0,0x20-0x40($ctx)  # b1.b0
 708         psrldq          \$8,$E0
 709         movq            @MSG0[0],0x60-0x40($ctx)# d1.d0
 710         movq            $E0,0x80-0x40($ctx)     # e1.e0
 711
 712         lea     `$REG_SZ/2`($ctx),$ctx
 713         lea     `16*2`($inp),$inp
 714         dec     $num
 715         jnz     .Loop_grande_shaext
 716
 717 .Ldone_shaext:
 718         #mov    `$REG_SZ*17`(%rsp),%rax         # original %rsp
 719 ___
 720 $code.=<<___ if ($win64);
 721         movaps  -0xb8(%rax),%xmm6
 722         movaps  -0xa8(%rax),%xmm7
 723         movaps  -0x98(%rax),%xmm8
 724         movaps  -0x88(%rax),%xmm9
 725         movaps  -0x78(%rax),%xmm10
 726         movaps  -0x68(%rax),%xmm11
 727         movaps  -0x58(%rax),%xmm12
 728         movaps  -0x48(%rax),%xmm13
 729         movaps  -0x38(%rax),%xmm14
 730         movaps  -0x28(%rax),%xmm15
 731 ___
 732 $code.=<<___;
 733         mov     -16(%rax),%rbp
 734         mov     -8(%rax),%rbx
 735         lea     (%rax),%rsp
 736 .Lepilogue_shaext:
 737         ret
 738 .size   sha1_multi_block_shaext,.-sha1_multi_block_shaext
 739 ___
 740                                                 }}}
 741
 742                                                 if ($avx) {{{
 743 sub BODY_00_19_avx {
 744 my ($i,$a,$b,$c,$d,$e)=@_;
 745 my $j=$i+1;
 746 my $k=$i+2;
 747 my $vpack = $REG_SZ==16 ? "vpunpckldq" : "vinserti128";
 748 my $ptr_n = $REG_SZ==16 ? @ptr[1] : @ptr[4];
 749
 750 $code.=<<___ if ($i==0 && $REG_SZ==16);
 751         vmovd           (@ptr[0]),@Xi[0]
 752          lea            `16*4`(@ptr[0]),@ptr[0]
 753         vmovd           (@ptr[1]),@Xi[2]        # borrow Xi[2]
 754          lea            `16*4`(@ptr[1]),@ptr[1]
 755         vpinsrd         \$1,(@ptr[2]),@Xi[0],@Xi[0]
 756          lea            `16*4`(@ptr[2]),@ptr[2]
 757         vpinsrd         \$1,(@ptr[3]),@Xi[2],@Xi[2]
 758          lea            `16*4`(@ptr[3]),@ptr[3]
 759          vmovd          `4*$j-16*4`(@ptr[0]),@Xi[1]
 760         vpunpckldq      @Xi[2],@Xi[0],@Xi[0]
 761          vmovd          `4*$j-16*4`($ptr_n),$t3
 762         vpshufb         $tx,@Xi[0],@Xi[0]
 763 ___
 764 $code.=<<___ if ($i<15 && $REG_SZ==16);         # just load input
 765          vpinsrd        \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
 766          vpinsrd        \$1,`4*$j-16*4`(@ptr[3]),$t3,$t3
 767 ___
 768 $code.=<<___ if ($i==0 && $REG_SZ==32);
 769         vmovd           (@ptr[0]),@Xi[0]
 770          lea            `16*4`(@ptr[0]),@ptr[0]
 771         vmovd           (@ptr[4]),@Xi[2]        # borrow Xi[2]
 772          lea            `16*4`(@ptr[4]),@ptr[4]
 773         vmovd           (@ptr[1]),$t2
 774          lea            `16*4`(@ptr[1]),@ptr[1]
 775         vmovd           (@ptr[5]),$t1
 776          lea            `16*4`(@ptr[5]),@ptr[5]
 777         vpinsrd         \$1,(@ptr[2]),@Xi[0],@Xi[0]
 778          lea            `16*4`(@ptr[2]),@ptr[2]
 779         vpinsrd         \$1,(@ptr[6]),@Xi[2],@Xi[2]
 780          lea            `16*4`(@ptr[6]),@ptr[6]
 781         vpinsrd         \$1,(@ptr[3]),$t2,$t2
 782          lea            `16*4`(@ptr[3]),@ptr[3]
 783         vpunpckldq      $t2,@Xi[0],@Xi[0]
 784         vpinsrd         \$1,(@ptr[7]),$t1,$t1
 785          lea            `16*4`(@ptr[7]),@ptr[7]
 786         vpunpckldq      $t1,@Xi[2],@Xi[2]
 787          vmovd          `4*$j-16*4`(@ptr[0]),@Xi[1]
 788         vinserti128     @Xi[2],@Xi[0],@Xi[0]
 789          vmovd          `4*$j-16*4`($ptr_n),$t3
 790         vpshufb         $tx,@Xi[0],@Xi[0]
 791 ___
 792 $code.=<<___ if ($i<15 && $REG_SZ==32);         # just load input
 793          vmovd          `4*$j-16*4`(@ptr[1]),$t2
 794          vmovd          `4*$j-16*4`(@ptr[5]),$t1
 795          vpinsrd        \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
 796          vpinsrd        \$1,`4*$j-16*4`(@ptr[6]),$t3,$t3
 797          vpinsrd        \$1,`4*$j-16*4`(@ptr[3]),$t2,$t2
 798          vpunpckldq     $t2,@Xi[1],@Xi[1]
 799          vpinsrd        \$1,`4*$j-16*4`(@ptr[7]),$t1,$t1
 800          vpunpckldq     $t1,$t3,$t3
 801 ___
 802 $code.=<<___ if ($i<14);
 803         vpaddd  $K,$e,$e                        # e+=K_00_19
 804         vpslld  \$5,$a,$t2
 805         vpandn  $d,$b,$t1
 806         vpand   $c,$b,$t0
 807
 808         vmovdqa @Xi[0],`&Xi_off($i)`
 809         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 810          $vpack         $t3,@Xi[1],@Xi[1]
 811         vpsrld  \$27,$a,$t3
 812         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 813          vmovd          `4*$k-16*4`(@ptr[0]),@Xi[2]
 814
 815         vpslld  \$30,$b,$t1
 816         vpor    $t3,$t2,$t2                     # rol(a,5)
 817          vmovd          `4*$k-16*4`($ptr_n),$t3
 818         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 819
 820         vpsrld  \$2,$b,$b
 821         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 822          vpshufb        $tx,@Xi[1],@Xi[1]
 823         vpor    $t1,$b,$b                       # b=rol(b,30)
 824 ___
 825 $code.=<<___ if ($i==14);
 826         vpaddd  $K,$e,$e                        # e+=K_00_19
 827          prefetcht0     63(@ptr[0])
 828         vpslld  \$5,$a,$t2
 829         vpandn  $d,$b,$t1
 830         vpand   $c,$b,$t0
 831
 832         vmovdqa @Xi[0],`&Xi_off($i)`
 833         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 834          $vpack         $t3,@Xi[1],@Xi[1]
 835         vpsrld  \$27,$a,$t3
 836          prefetcht0     63(@ptr[1])
 837         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 838
 839         vpslld  \$30,$b,$t1
 840         vpor    $t3,$t2,$t2                     # rol(a,5)
 841          prefetcht0     63(@ptr[2])
 842         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 843
 844         vpsrld  \$2,$b,$b
 845         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 846          prefetcht0     63(@ptr[3])
 847          vpshufb        $tx,@Xi[1],@Xi[1]
 848         vpor    $t1,$b,$b                       # b=rol(b,30)
 849 ___
 850 $code.=<<___ if ($i>=13 && $i<15);
 851         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # preload "X[2]"
 852 ___
 853 $code.=<<___ if ($i>=15);                       # apply Xupdate
 854         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 855         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 856
 857         vpaddd  $K,$e,$e                        # e+=K_00_19
 858         vpslld  \$5,$a,$t2
 859         vpandn  $d,$b,$t1
 860          `"prefetcht0   63(@ptr[4])"            if ($i==15 && $REG_SZ==32)`
 861         vpand   $c,$b,$t0
 862
 863         vmovdqa @Xi[0],`&Xi_off($i)`
 864         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 865          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 866         vpsrld  \$27,$a,$t3
 867         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 868          vpxor  @Xi[3],@Xi[1],@Xi[1]
 869          `"prefetcht0   63(@ptr[5])"            if ($i==15 && $REG_SZ==32)`
 870
 871         vpslld  \$30,$b,$t1
 872         vpor    $t3,$t2,$t2                     # rol(a,5)
 873         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 874          `"prefetcht0   63(@ptr[6])"            if ($i==15 && $REG_SZ==32)`
 875          vpsrld \$31,@Xi[1],$tx
 876          vpaddd @Xi[1],@Xi[1],@Xi[1]
 877
 878         vpsrld  \$2,$b,$b
 879          `"prefetcht0   63(@ptr[7])"            if ($i==15 && $REG_SZ==32)`
 880         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 881          vpor   $tx,@Xi[1],@Xi[1]               # rol   \$1,@Xi[1]
 882         vpor    $t1,$b,$b                       # b=rol(b,30)
 883 ___
 884 push(@Xi,shift(@Xi));
 885 }
 886
 887 sub BODY_20_39_avx {
 888 my ($i,$a,$b,$c,$d,$e)=@_;
 889 my $j=$i+1;
 890
 891 $code.=<<___ if ($i<79);
 892         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 893         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 894
 895         vpslld  \$5,$a,$t2
 896         vpaddd  $K,$e,$e                        # e+=K_20_39
 897         vpxor   $b,$d,$t0
 898 ___
 899 $code.=<<___ if ($i<72);
 900         vmovdqa @Xi[0],`&Xi_off($i)`
 901 ___
 902 $code.=<<___ if ($i<79);
 903         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 904          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 905         vpsrld  \$27,$a,$t3
 906         vpxor   $c,$t0,$t0                      # Parity(b,c,d)
 907          vpxor  @Xi[3],@Xi[1],@Xi[1]
 908
 909         vpslld  \$30,$b,$t1
 910         vpor    $t3,$t2,$t2                     # rol(a,5)
 911         vpaddd  $t0,$e,$e                       # e+=Parity(b,c,d)
 912          vpsrld \$31,@Xi[1],$tx
 913          vpaddd @Xi[1],@Xi[1],@Xi[1]
 914
 915         vpsrld  \$2,$b,$b
 916         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 917          vpor   $tx,@Xi[1],@Xi[1]               # rol(@Xi[1],1)
 918         vpor    $t1,$b,$b                       # b=rol(b,30)
 919 ___
 920 $code.=<<___ if ($i==79);
 921         vpslld  \$5,$a,$t2
 922         vpaddd  $K,$e,$e                        # e+=K_20_39
 923         vpxor   $b,$d,$t0
 924
 925         vpsrld  \$27,$a,$t3
 926         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 927         vpxor   $c,$t0,$t0                      # Parity(b,c,d)
 928
 929         vpslld  \$30,$b,$t1
 930         vpor    $t3,$t2,$t2                     # rol(a,5)
 931         vpaddd  $t0,$e,$e                       # e+=Parity(b,c,d)
 932
 933         vpsrld  \$2,$b,$b
 934         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 935         vpor    $t1,$b,$b                       # b=rol(b,30)
 936 ___
 937 push(@Xi,shift(@Xi));
 938 }
 939
 940 sub BODY_40_59_avx {
 941 my ($i,$a,$b,$c,$d,$e)=@_;
 942 my $j=$i+1;
 943
 944 $code.=<<___;
 945         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 946         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 947
 948         vpaddd  $K,$e,$e                        # e+=K_40_59
 949         vpslld  \$5,$a,$t2
 950         vpand   $c,$d,$t1
 951          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 952
 953         vpaddd  $t1,$e,$e
 954         vpsrld  \$27,$a,$t3
 955         vpxor   $c,$d,$t0
 956          vpxor  @Xi[3],@Xi[1],@Xi[1]
 957
 958         vmovdqu @Xi[0],`&Xi_off($i)`
 959         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 960         vpor    $t3,$t2,$t2                     # rol(a,5)
 961          vpsrld \$31,@Xi[1],$tx
 962         vpand   $b,$t0,$t0
 963          vpaddd @Xi[1],@Xi[1],@Xi[1]
 964
 965         vpslld  \$30,$b,$t1
 966         vpaddd  $t0,$e,$e                       # e+=Maj(b,d,c)
 967
 968         vpsrld  \$2,$b,$b
 969         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 970          vpor   $tx,@Xi[1],@Xi[1]               # rol(@X[1],1)
 971         vpor    $t1,$b,$b                       # b=rol(b,30)
 972 ___
 973 push(@Xi,shift(@Xi));
 974 }
 975
 976 $code.=<<___;
 977 .type   sha1_multi_block_avx,\@function,3
 978 .align  32
 979 sha1_multi_block_avx:
 980 _avx_shortcut:
 981 ___
 982 $code.=<<___ if ($avx>1);
 983         shr     \$32,%rcx
 984         cmp     \$2,$num
 985         jb      .Lavx
 986         test    \$`1<<5`,%ecx
 987         jnz     _avx2_shortcut
 988         jmp     .Lavx
 989 .align  32
 990 .Lavx:
 991 ___
 992 $code.=<<___;
 993         mov     %rsp,%rax
 994         push    %rbx
 995         push    %rbp
 996 ___
 997 $code.=<<___ if ($win64);
 998         lea     -0xa8(%rsp),%rsp
 999         movaps  %xmm6,(%rsp)
1000         movaps  %xmm7,0x10(%rsp)
1001         movaps  %xmm8,0x20(%rsp)
1002         movaps  %xmm9,0x30(%rsp)
1003         movaps  %xmm10,-0x78(%rax)
1004         movaps  %xmm11,-0x68(%rax)
1005         movaps  %xmm12,-0x58(%rax)
1006         movaps  %xmm13,-0x48(%rax)
1007         movaps  %xmm14,-0x38(%rax)
1008         movaps  %xmm15,-0x28(%rax)
1009 ___
1010 $code.=<<___;
1011         sub     \$`$REG_SZ*18`, %rsp
1012         and     \$-256,%rsp
1013         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1014 .Lbody_avx:
1015         lea     K_XX_XX(%rip),$Tbl
1016         lea     `$REG_SZ*16`(%rsp),%rbx
1017
1018         vzeroupper
1019 .Loop_grande_avx:
1020         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1021         xor     $num,$num
1022 ___
1023 for($i=0;$i<4;$i++) {
1024     $code.=<<___;
1025         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1026         mov     `16*$i+8`($inp),%ecx            # number of blocks
1027         cmp     $num,%ecx
1028         cmovg   %ecx,$num                       # find maximum
1029         test    %ecx,%ecx
1030         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1031         cmovle  $Tbl,@ptr[$i]                   # cancel input
1032 ___
1033 }
1034 $code.=<<___;
1035         test    $num,$num
1036         jz      .Ldone_avx
1037
1038         vmovdqu 0x00($ctx),$A                   # load context
1039          lea    128(%rsp),%rax
1040         vmovdqu 0x20($ctx),$B
1041         vmovdqu 0x40($ctx),$C
1042         vmovdqu 0x60($ctx),$D
1043         vmovdqu 0x80($ctx),$E
1044         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1045         jmp     .Loop_avx
1046
1047 .align  32
1048 .Loop_avx:
1049 ___
1050 $code.="        vmovdqa -0x20($Tbl),$K\n";      # K_00_19
1051 for($i=0;$i<20;$i++)    { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1052 $code.="        vmovdqa 0x00($Tbl),$K\n";       # K_20_39
1053 for(;$i<40;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1054 $code.="        vmovdqa 0x20($Tbl),$K\n";       # K_40_59
1055 for(;$i<60;$i++)        { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1056 $code.="        vmovdqa 0x40($Tbl),$K\n";       # K_60_79
1057 for(;$i<80;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1058 $code.=<<___;
1059         mov     \$1,%ecx
1060 ___
1061 for($i=0;$i<4;$i++) {
1062     $code.=<<___;
1063         cmp     `4*$i`(%rbx),%ecx               # examine counters
1064         cmovge  $Tbl,@ptr[$i]                   # cancel input
1065 ___
1066 }
1067 $code.=<<___;
1068         vmovdqu (%rbx),$t0                      # pull counters
1069         vpxor   $t2,$t2,$t2
1070         vmovdqa $t0,$t1
1071         vpcmpgtd $t2,$t1,$t1                    # mask value
1072         vpaddd  $t1,$t0,$t0                     # counters--
1073
1074         vpand   $t1,$A,$A
1075         vpand   $t1,$B,$B
1076         vpaddd  0x00($ctx),$A,$A
1077         vpand   $t1,$C,$C
1078         vpaddd  0x20($ctx),$B,$B
1079         vpand   $t1,$D,$D
1080         vpaddd  0x40($ctx),$C,$C
1081         vpand   $t1,$E,$E
1082         vpaddd  0x60($ctx),$D,$D
1083         vpaddd  0x80($ctx),$E,$E
1084         vmovdqu $A,0x00($ctx)
1085         vmovdqu $B,0x20($ctx)
1086         vmovdqu $C,0x40($ctx)
1087         vmovdqu $D,0x60($ctx)
1088         vmovdqu $E,0x80($ctx)
1089
1090         vmovdqu $t0,(%rbx)                      # save counters
1091         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1092         dec     $num
1093         jnz     .Loop_avx
1094
1095         mov     `$REG_SZ*17+8`(%rsp),$num
1096         lea     $REG_SZ($ctx),$ctx
1097         lea     `16*$REG_SZ/4`($inp),$inp
1098         dec     $num
1099         jnz     .Loop_grande_avx
1100
1101 .Ldone_avx:
1102         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
1103         vzeroupper
1104 ___
1105 $code.=<<___ if ($win64);
1106         movaps  -0xb8(%rax),%xmm6
1107         movaps  -0xa8(%rax),%xmm7
1108         movaps  -0x98(%rax),%xmm8
1109         movaps  -0x88(%rax),%xmm9
1110         movaps  -0x78(%rax),%xmm10
1111         movaps  -0x68(%rax),%xmm11
1112         movaps  -0x58(%rax),%xmm12
1113         movaps  -0x48(%rax),%xmm13
1114         movaps  -0x38(%rax),%xmm14
1115         movaps  -0x28(%rax),%xmm15
1116 ___
1117 $code.=<<___;
1118         mov     -16(%rax),%rbp
1119         mov     -8(%rax),%rbx
1120         lea     (%rax),%rsp
1121 .Lepilogue_avx:
1122         ret
1123 .size   sha1_multi_block_avx,.-sha1_multi_block_avx
1124 ___
1125
1126                                                 if ($avx>1) {
1127 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1128
1129 $REG_SZ=32;
1130
1131 @ptr=map("%r$_",(12..15,8..11));
1132
1133 @V=($A,$B,$C,$D,$E)=map("%ymm$_",(0..4));
1134 ($t0,$t1,$t2,$t3,$tx)=map("%ymm$_",(5..9));
1135 @Xi=map("%ymm$_",(10..14));
1136 $K="%ymm15";
1137
1138 $code.=<<___;
1139 .type   sha1_multi_block_avx2,\@function,3
1140 .align  32
1141 sha1_multi_block_avx2:
1142 _avx2_shortcut:
1143         mov     %rsp,%rax
1144         push    %rbx
1145         push    %rbp
1146         push    %r12
1147         push    %r13
1148         push    %r14
1149         push    %r15
1150 ___
1151 $code.=<<___ if ($win64);
1152         lea     -0xa8(%rsp),%rsp
1153         movaps  %xmm6,(%rsp)
1154         movaps  %xmm7,0x10(%rsp)
1155         movaps  %xmm8,0x20(%rsp)
1156         movaps  %xmm9,0x30(%rsp)
1157         movaps  %xmm10,0x40(%rsp)
1158         movaps  %xmm11,0x50(%rsp)
1159         movaps  %xmm12,-0x78(%rax)
1160         movaps  %xmm13,-0x68(%rax)
1161         movaps  %xmm14,-0x58(%rax)
1162         movaps  %xmm15,-0x48(%rax)
1163 ___
1164 $code.=<<___;
1165         sub     \$`$REG_SZ*18`, %rsp
1166         and     \$-256,%rsp
1167         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1168 .Lbody_avx2:
1169         lea     K_XX_XX(%rip),$Tbl
1170         shr     \$1,$num
1171
1172         vzeroupper
1173 .Loop_grande_avx2:
1174         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1175         xor     $num,$num
1176         lea     `$REG_SZ*16`(%rsp),%rbx
1177 ___
1178 for($i=0;$i<8;$i++) {
1179     $code.=<<___;
1180         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1181         mov     `16*$i+8`($inp),%ecx            # number of blocks
1182         cmp     $num,%ecx
1183         cmovg   %ecx,$num                       # find maximum
1184         test    %ecx,%ecx
1185         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1186         cmovle  $Tbl,@ptr[$i]                   # cancel input
1187 ___
1188 }
1189 $code.=<<___;
1190         vmovdqu 0x00($ctx),$A                   # load context
1191          lea    128(%rsp),%rax
1192         vmovdqu 0x20($ctx),$B
1193          lea    256+128(%rsp),%rbx
1194         vmovdqu 0x40($ctx),$C
1195         vmovdqu 0x60($ctx),$D
1196         vmovdqu 0x80($ctx),$E
1197         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1198         jmp     .Loop_avx2
1199
1200 .align  32
1201 .Loop_avx2:
1202 ___
1203 $code.="        vmovdqa -0x20($Tbl),$K\n";      # K_00_19
1204 for($i=0;$i<20;$i++)    { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1205 $code.="        vmovdqa 0x00($Tbl),$K\n";       # K_20_39
1206 for(;$i<40;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1207 $code.="        vmovdqa 0x20($Tbl),$K\n";       # K_40_59
1208 for(;$i<60;$i++)        { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1209 $code.="        vmovdqa 0x40($Tbl),$K\n";       # K_60_79
1210 for(;$i<80;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1211 $code.=<<___;
1212         mov     \$1,%ecx
1213         lea     `$REG_SZ*16`(%rsp),%rbx
1214 ___
1215 for($i=0;$i<8;$i++) {
1216     $code.=<<___;
1217         cmp     `4*$i`(%rbx),%ecx               # examine counters
1218         cmovge  $Tbl,@ptr[$i]                   # cancel input
1219 ___
1220 }
1221 $code.=<<___;
1222         vmovdqu (%rbx),$t0              # pull counters
1223         vpxor   $t2,$t2,$t2
1224         vmovdqa $t0,$t1
1225         vpcmpgtd $t2,$t1,$t1                    # mask value
1226         vpaddd  $t1,$t0,$t0                     # counters--
1227
1228         vpand   $t1,$A,$A
1229         vpand   $t1,$B,$B
1230         vpaddd  0x00($ctx),$A,$A
1231         vpand   $t1,$C,$C
1232         vpaddd  0x20($ctx),$B,$B
1233         vpand   $t1,$D,$D
1234         vpaddd  0x40($ctx),$C,$C
1235         vpand   $t1,$E,$E
1236         vpaddd  0x60($ctx),$D,$D
1237         vpaddd  0x80($ctx),$E,$E
1238         vmovdqu $A,0x00($ctx)
1239         vmovdqu $B,0x20($ctx)
1240         vmovdqu $C,0x40($ctx)
1241         vmovdqu $D,0x60($ctx)
1242         vmovdqu $E,0x80($ctx)
1243
1244         vmovdqu $t0,(%rbx)                      # save counters
1245         lea     256+128(%rsp),%rbx
1246         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1247         dec     $num
1248         jnz     .Loop_avx2
1249
1250         #mov    `$REG_SZ*17+8`(%rsp),$num
1251         #lea    $REG_SZ($ctx),$ctx
1252         #lea    `16*$REG_SZ/4`($inp),$inp
1253         #dec    $num
1254         #jnz    .Loop_grande_avx2
1255
1256 .Ldone_avx2:
1257         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
1258         vzeroupper
1259 ___
1260 $code.=<<___ if ($win64);
1261         movaps  -0xd8(%rax),%xmm6
1262         movaps  -0xc8(%rax),%xmm7
1263         movaps  -0xb8(%rax),%xmm8
1264         movaps  -0xa8(%rax),%xmm9
1265         movaps  -0x98(%rax),%xmm10
1266         movaps  -0x88(%rax),%xmm11
1267         movaps  -0x78(%rax),%xmm12
1268         movaps  -0x68(%rax),%xmm13
1269         movaps  -0x58(%rax),%xmm14
1270         movaps  -0x48(%rax),%xmm15
1271 ___
1272 $code.=<<___;
1273         mov     -48(%rax),%r15
1274         mov     -40(%rax),%r14
1275         mov     -32(%rax),%r13
1276         mov     -24(%rax),%r12
1277         mov     -16(%rax),%rbp
1278         mov     -8(%rax),%rbx
1279         lea     (%rax),%rsp
1280 .Lepilogue_avx2:
1281         ret
1282 .size   sha1_multi_block_avx2,.-sha1_multi_block_avx2
1283 ___
1284                                                 }       }}}
1285 $code.=<<___;
1286
1287 .align  256
1288         .long   0x5a827999,0x5a827999,0x5a827999,0x5a827999     # K_00_19
1289         .long   0x5a827999,0x5a827999,0x5a827999,0x5a827999     # K_00_19
1290 K_XX_XX:
1291         .long   0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1     # K_20_39
1292         .long   0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1     # K_20_39
1293         .long   0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc     # K_40_59
1294         .long   0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc     # K_40_59
1295         .long   0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6     # K_60_79
1296         .long   0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6     # K_60_79
1297         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1298         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1299         .byte   0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0
1300         .asciz  "SHA1 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
1301 ___
1302
1303 if ($win64) {
1304 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1305 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1306 $rec="%rcx";
1307 $frame="%rdx";
1308 $context="%r8";
1309 $disp="%r9";
1310
1311 $code.=<<___;
1312 .extern __imp_RtlVirtualUnwind
1313 .type   se_handler,\@abi-omnipotent
1314 .align  16
1315 se_handler:
1316         push    %rsi
1317         push    %rdi
1318         push    %rbx
1319         push    %rbp
1320         push    %r12
1321         push    %r13
1322         push    %r14
1323         push    %r15
1324         pushfq
1325         sub     \$64,%rsp
1326
1327         mov     120($context),%rax      # pull context->Rax
1328         mov     248($context),%rbx      # pull context->Rip
1329
1330         mov     8($disp),%rsi           # disp->ImageBase
1331         mov     56($disp),%r11          # disp->HandlerData
1332
1333         mov     0(%r11),%r10d           # HandlerData[0]
1334         lea     (%rsi,%r10),%r10        # end of prologue label
1335         cmp     %r10,%rbx               # context->Rip<.Lbody
1336         jb      .Lin_prologue
1337
1338         mov     152($context),%rax      # pull context->Rsp
1339
1340         mov     4(%r11),%r10d           # HandlerData[1]
1341         lea     (%rsi,%r10),%r10        # epilogue label
1342         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1343         jae     .Lin_prologue
1344
1345         mov     `16*17`(%rax),%rax      # pull saved stack pointer
1346
1347         mov     -8(%rax),%rbx
1348         mov     -16(%rax),%rbp
1349         mov     %rbx,144($context)      # restore context->Rbx
1350         mov     %rbp,160($context)      # restore context->Rbp
1351
1352         lea     -24-10*16(%rax),%rsi
1353         lea     512($context),%rdi      # &context.Xmm6
1354         mov     \$20,%ecx
1355         .long   0xa548f3fc              # cld; rep movsq
1356
1357 .Lin_prologue:
1358         mov     8(%rax),%rdi
1359         mov     16(%rax),%rsi
1360         mov     %rax,152($context)      # restore context->Rsp
1361         mov     %rsi,168($context)      # restore context->Rsi
1362         mov     %rdi,176($context)      # restore context->Rdi
1363
1364         mov     40($disp),%rdi          # disp->ContextRecord
1365         mov     $context,%rsi           # context
1366         mov     \$154,%ecx              # sizeof(CONTEXT)
1367         .long   0xa548f3fc              # cld; rep movsq
1368
1369         mov     $disp,%rsi
1370         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1371         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1372         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1373         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1374         mov     40(%rsi),%r10           # disp->ContextRecord
1375         lea     56(%rsi),%r11           # &disp->HandlerData
1376         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1377         mov     %r10,32(%rsp)           # arg5
1378         mov     %r11,40(%rsp)           # arg6
1379         mov     %r12,48(%rsp)           # arg7
1380         mov     %rcx,56(%rsp)           # arg8, (NULL)
1381         call    *__imp_RtlVirtualUnwind(%rip)
1382
1383         mov     \$1,%eax                # ExceptionContinueSearch
1384         add     \$64,%rsp
1385         popfq
1386         pop     %r15
1387         pop     %r14
1388         pop     %r13
1389         pop     %r12
1390         pop     %rbp
1391         pop     %rbx
1392         pop     %rdi
1393         pop     %rsi
1394         ret
1395 .size   se_handler,.-se_handler
1396 ___
1397 $code.=<<___ if ($avx>1);
1398 .type   avx2_handler,\@abi-omnipotent
1399 .align  16
1400 avx2_handler:
1401         push    %rsi
1402         push    %rdi
1403         push    %rbx
1404         push    %rbp
1405         push    %r12
1406         push    %r13
1407         push    %r14
1408         push    %r15
1409         pushfq
1410         sub     \$64,%rsp
1411
1412         mov     120($context),%rax      # pull context->Rax
1413         mov     248($context),%rbx      # pull context->Rip
1414
1415         mov     8($disp),%rsi           # disp->ImageBase
1416         mov     56($disp),%r11          # disp->HandlerData
1417
1418         mov     0(%r11),%r10d           # HandlerData[0]
1419         lea     (%rsi,%r10),%r10        # end of prologue label
1420         cmp     %r10,%rbx               # context->Rip<body label
1421         jb      .Lin_prologue
1422
1423         mov     152($context),%rax      # pull context->Rsp
1424
1425         mov     4(%r11),%r10d           # HandlerData[1]
1426         lea     (%rsi,%r10),%r10        # epilogue label
1427         cmp     %r10,%rbx               # context->Rip>=epilogue label
1428         jae     .Lin_prologue
1429
1430         mov     `32*17`($context),%rax  # pull saved stack pointer
1431
1432         mov     -8(%rax),%rbx
1433         mov     -16(%rax),%rbp
1434         mov     -24(%rax),%r12
1435         mov     -32(%rax),%r13
1436         mov     -40(%rax),%r14
1437         mov     -48(%rax),%r15
1438         mov     %rbx,144($context)      # restore context->Rbx
1439         mov     %rbp,160($context)      # restore context->Rbp
1440         mov     %r12,216($context)      # restore cotnext->R12
1441         mov     %r13,224($context)      # restore cotnext->R13
1442         mov     %r14,232($context)      # restore cotnext->R14
1443         mov     %r15,240($context)      # restore cotnext->R15
1444
1445         lea     -56-10*16(%rax),%rsi
1446         lea     512($context),%rdi      # &context.Xmm6
1447         mov     \$20,%ecx
1448         .long   0xa548f3fc              # cld; rep movsq
1449
1450         jmp     .Lin_prologue
1451 .size   avx2_handler,.-avx2_handler
1452 ___
1453 $code.=<<___;
1454 .section        .pdata
1455 .align  4
1456         .rva    .LSEH_begin_sha1_multi_block
1457         .rva    .LSEH_end_sha1_multi_block
1458         .rva    .LSEH_info_sha1_multi_block
1459         .rva    .LSEH_begin_sha1_multi_block_shaext
1460         .rva    .LSEH_end_sha1_multi_block_shaext
1461         .rva    .LSEH_info_sha1_multi_block_shaext
1462 ___
1463 $code.=<<___ if ($avx);
1464         .rva    .LSEH_begin_sha1_multi_block_avx
1465         .rva    .LSEH_end_sha1_multi_block_avx
1466         .rva    .LSEH_info_sha1_multi_block_avx
1467 ___
1468 $code.=<<___ if ($avx>1);
1469         .rva    .LSEH_begin_sha1_multi_block_avx2
1470         .rva    .LSEH_end_sha1_multi_block_avx2
1471         .rva    .LSEH_info_sha1_multi_block_avx2
1472 ___
1473 $code.=<<___;
1474 .section        .xdata
1475 .align  8
1476 .LSEH_info_sha1_multi_block:
1477         .byte   9,0,0,0
1478         .rva    se_handler
1479         .rva    .Lbody,.Lepilogue                       # HandlerData[]
1480 .LSEH_info_sha1_multi_block_shaext:
1481         .byte   9,0,0,0
1482         .rva    se_handler
1483         .rva    .Lbody_shaext,.Lepilogue_shaext # HandlerData[]
1484 ___
1485 $code.=<<___ if ($avx);
1486 .LSEH_info_sha1_multi_block_avx:
1487         .byte   9,0,0,0
1488         .rva    se_handler
1489         .rva    .Lbody_avx,.Lepilogue_avx               # HandlerData[]
1490 ___
1491 $code.=<<___ if ($avx>1);
1492 .LSEH_info_sha1_multi_block_avx2:
1493         .byte   9,0,0,0
1494         .rva    avx2_handler
1495         .rva    .Lbody_avx2,.Lepilogue_avx2             # HandlerData[]
1496 ___
1497 }
1498 ####################################################################
1499
1500 sub rex {
1501   local *opcode=shift;
1502   my ($dst,$src)=@_;
1503   my $rex=0;
1504
1505     $rex|=0x04                  if ($dst>=8);
1506     $rex|=0x01                  if ($src>=8);
1507     unshift @opcode,$rex|0x40   if ($rex);
1508 }
1509
1510 sub sha1rnds4 {
1511     if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1512       my @opcode=(0x0f,0x3a,0xcc);
1513         rex(\@opcode,$3,$2);
1514         push @opcode,0xc0|($2&7)|(($3&7)<<3);           # ModR/M
1515         my $c=$1;
1516         push @opcode,$c=~/^0/?oct($c):$c;
1517         return ".byte\t".join(',',@opcode);
1518     } else {
1519         return "sha1rnds4\t".@_[0];
1520     }
1521 }
1522
1523 sub sha1op38 {
1524     my $instr = shift;
1525     my %opcodelet = (
1526                 "sha1nexte" => 0xc8,
1527                 "sha1msg1"  => 0xc9,
1528                 "sha1msg2"  => 0xca     );
1529
1530     if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1531       my @opcode=(0x0f,0x38);
1532         rex(\@opcode,$2,$1);
1533         push @opcode,$opcodelet{$instr};
1534         push @opcode,0xc0|($1&7)|(($2&7)<<3);           # ModR/M
1535         return ".byte\t".join(',',@opcode);
1536     } else {
1537         return $instr."\t".@_[0];
1538     }
1539 }
1540
1541 foreach (split("\n",$code)) {
1542         s/\`([^\`]*)\`/eval($1)/ge;
1543
1544         s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo                or
1545         s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo             or
1546
1547         s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go          or
1548         s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go         or
1549         s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go    or
1550         s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go        or
1551         s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go            or
1552         s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;
1553
1554         print $_,"\n";
1555 }
1556
1557 close STDOUT;