crypto/sha/asm/sha1-mb-x86_64.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9
  10 # Multi-buffer SHA1 procedure processes n buffers in parallel by
  11 # placing buffer data to designated lane of SIMD register. n is
  12 # naturally limited to 4 on pre-AVX2 processors and to 8 on
  13 # AVX2-capable processors such as Haswell.
  14 #
  15 #               this    +aesni(i)       sha1    aesni-sha1      gain(iv)
  16 # -------------------------------------------------------------------
  17 # Westmere(ii)  10.7/n  +1.28=3.96(n=4) 5.30    6.66            +68%
  18 # Atom(ii)      18.1/n  +3.93=8.46(n=4) 9.37    12.8            +51%
  19 # Sandy Bridge  (8.16   +5.15=13.3)/n   4.99    5.98            +80%
  20 # Ivy Bridge    (8.08   +5.14=13.2)/n   4.60    5.54            +68%
  21 # Haswell(iii)  (8.96   +5.00=14.0)/n   3.57    4.55            +160%
  22 # Skylake       (8.70   +5.00=13.7)/n   3.64    4.20            +145%
  23 # Bulldozer     (9.76   +5.76=15.5)/n   5.95    6.37            +64%
  24 #
  25 # (i)   multi-block CBC encrypt with 128-bit key;
  26 # (ii)  (HASH+AES)/n does not apply to Westmere for n>3 and Atom,
  27 #       because of lower AES-NI instruction throughput;
  28 # (iii) "this" is for n=8, when we gather twice as much data, result
  29 #       for n=4 is 8.00+4.44=12.4;
  30 # (iv)  presented improvement coefficients are asymptotic limits and
  31 #       in real-life application are somewhat lower, e.g. for 2KB
  32 #       fragments they range from 30% to 100% (on Haswell);
  33
  34 $flavour = shift;
  35 $output  = shift;
  36 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  37
  38 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  39
  40 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  41 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  42 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  43 die "can't locate x86_64-xlate.pl";
  44
  45 $avx=0;
  46
  47 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  48                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  49         $avx = ($1>=2.19) + ($1>=2.22);
  50 }
  51
  52 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  53            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  54         $avx = ($1>=2.09) + ($1>=2.10);
  55 }
  56
  57 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  58            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  59         $avx = ($1>=10) + ($1>=11);
  60 }
  61
  62 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|based on LLVM) ([3-9]\.[0-9]+)/) {
  63         $avx = ($2>=3.0) + ($2>3.0);
  64 }
  65
  66 open OUT,"| \"$^X\" $xlate $flavour $output";
  67 *STDOUT=*OUT;
  68
  69 # void sha1_multi_block (
  70 #     struct {  unsigned int A[8];
  71 #               unsigned int B[8];
  72 #               unsigned int C[8];
  73 #               unsigned int D[8];
  74 #               unsigned int E[8];      } *ctx,
  75 #     struct {  void *ptr; int blocks;  } inp[8],
  76 #     int num);         /* 1 or 2 */
  77 #
  78 $ctx="%rdi";    # 1st arg
  79 $inp="%rsi";    # 2nd arg
  80 $num="%edx";
  81 @ptr=map("%r$_",(8..11));
  82 $Tbl="%rbp";
  83
  84 @V=($A,$B,$C,$D,$E)=map("%xmm$_",(0..4));
  85 ($t0,$t1,$t2,$t3,$tx)=map("%xmm$_",(5..9));
  86 @Xi=map("%xmm$_",(10..14));
  87 $K="%xmm15";
  88
  89 if (1) {
  90     # Atom-specific optimization aiming to eliminate pshufb with high
  91     # registers [and thus get rid of 48 cycles accumulated penalty]
  92     @Xi=map("%xmm$_",(0..4));
  93     ($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9));
  94     @V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14));
  95 }
  96
  97 $REG_SZ=16;
  98
  99 sub Xi_off {
 100 my $off = shift;
 101
 102     $off %= 16; $off *= $REG_SZ;
 103     $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
 104 }
 105
 106 sub BODY_00_19 {
 107 my ($i,$a,$b,$c,$d,$e)=@_;
 108 my $j=$i+1;
 109 my $k=$i+2;
 110
 111 # Loads are performed 2+3/4 iterations in advance. 3/4 means that out
 112 # of 4 words you would expect to be loaded per given iteration one is
 113 # spilled to next iteration. In other words indices in four input
 114 # streams are distributed as following:
 115 #
 116 # $i==0:        0,0,0,0,1,1,1,1,2,2,2,
 117 # $i==1:        2,3,3,3,
 118 # $i==2:        3,4,4,4,
 119 # ...
 120 # $i==13:       14,15,15,15,
 121 # $i==14:       15
 122 #
 123 # Then at $i==15 Xupdate is applied one iteration in advance...
 124 $code.=<<___ if ($i==0);
 125         movd            (@ptr[0]),@Xi[0]
 126          lea            `16*4`(@ptr[0]),@ptr[0]
 127         movd            (@ptr[1]),@Xi[2]        # borrow @Xi[2]
 128          lea            `16*4`(@ptr[1]),@ptr[1]
 129         movd            (@ptr[2]),@Xi[3]        # borrow @Xi[3]
 130          lea            `16*4`(@ptr[2]),@ptr[2]
 131         movd            (@ptr[3]),@Xi[4]        # borrow @Xi[4]
 132          lea            `16*4`(@ptr[3]),@ptr[3]
 133         punpckldq       @Xi[3],@Xi[0]
 134          movd           `4*$j-16*4`(@ptr[0]),@Xi[1]
 135         punpckldq       @Xi[4],@Xi[2]
 136          movd           `4*$j-16*4`(@ptr[1]),$t3
 137         punpckldq       @Xi[2],@Xi[0]
 138          movd           `4*$j-16*4`(@ptr[2]),$t2
 139         pshufb          $tx,@Xi[0]
 140 ___
 141 $code.=<<___ if ($i<14);                        # just load input
 142          movd           `4*$j-16*4`(@ptr[3]),$t1
 143          punpckldq      $t2,@Xi[1]
 144         movdqa  $a,$t2
 145         paddd   $K,$e                           # e+=K_00_19
 146          punpckldq      $t1,$t3
 147         movdqa  $b,$t1
 148         movdqa  $b,$t0
 149         pslld   \$5,$t2
 150         pandn   $d,$t1
 151         pand    $c,$t0
 152          punpckldq      $t3,@Xi[1]
 153         movdqa  $a,$t3
 154
 155         movdqa  @Xi[0],`&Xi_off($i)`
 156         paddd   @Xi[0],$e                       # e+=X[i]
 157          movd           `4*$k-16*4`(@ptr[0]),@Xi[2]
 158         psrld   \$27,$t3
 159         pxor    $t1,$t0                         # Ch(b,c,d)
 160         movdqa  $b,$t1
 161
 162         por     $t3,$t2                         # rol(a,5)
 163          movd           `4*$k-16*4`(@ptr[1]),$t3
 164         pslld   \$30,$t1
 165         paddd   $t0,$e                          # e+=Ch(b,c,d)
 166
 167         psrld   \$2,$b
 168         paddd   $t2,$e                          # e+=rol(a,5)
 169          pshufb $tx,@Xi[1]
 170          movd           `4*$k-16*4`(@ptr[2]),$t2
 171         por     $t1,$b                          # b=rol(b,30)
 172 ___
 173 $code.=<<___ if ($i==14);                       # just load input
 174          movd           `4*$j-16*4`(@ptr[3]),$t1
 175          punpckldq      $t2,@Xi[1]
 176         movdqa  $a,$t2
 177         paddd   $K,$e                           # e+=K_00_19
 178          punpckldq      $t1,$t3
 179         movdqa  $b,$t1
 180         movdqa  $b,$t0
 181         pslld   \$5,$t2
 182          prefetcht0     63(@ptr[0])
 183         pandn   $d,$t1
 184         pand    $c,$t0
 185          punpckldq      $t3,@Xi[1]
 186         movdqa  $a,$t3
 187
 188         movdqa  @Xi[0],`&Xi_off($i)`
 189         paddd   @Xi[0],$e                       # e+=X[i]
 190         psrld   \$27,$t3
 191         pxor    $t1,$t0                         # Ch(b,c,d)
 192         movdqa  $b,$t1
 193          prefetcht0     63(@ptr[1])
 194
 195         por     $t3,$t2                         # rol(a,5)
 196         pslld   \$30,$t1
 197         paddd   $t0,$e                          # e+=Ch(b,c,d)
 198          prefetcht0     63(@ptr[2])
 199
 200         psrld   \$2,$b
 201         paddd   $t2,$e                          # e+=rol(a,5)
 202          pshufb $tx,@Xi[1]
 203          prefetcht0     63(@ptr[3])
 204         por     $t1,$b                          # b=rol(b,30)
 205 ___
 206 $code.=<<___ if ($i>=13 && $i<15);
 207         movdqa  `&Xi_off($j+2)`,@Xi[3]          # preload "X[2]"
 208 ___
 209 $code.=<<___ if ($i>=15);                       # apply Xupdate
 210         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 211         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 212
 213         movdqa  $a,$t2
 214          pxor   `&Xi_off($j+8)`,@Xi[1]
 215         paddd   $K,$e                           # e+=K_00_19
 216         movdqa  $b,$t1
 217         pslld   \$5,$t2
 218          pxor   @Xi[3],@Xi[1]
 219         movdqa  $b,$t0
 220         pandn   $d,$t1
 221          movdqa @Xi[1],$tx
 222         pand    $c,$t0
 223         movdqa  $a,$t3
 224          psrld  \$31,$tx
 225          paddd  @Xi[1],@Xi[1]
 226
 227         movdqa  @Xi[0],`&Xi_off($i)`
 228         paddd   @Xi[0],$e                       # e+=X[i]
 229         psrld   \$27,$t3
 230         pxor    $t1,$t0                         # Ch(b,c,d)
 231
 232         movdqa  $b,$t1
 233         por     $t3,$t2                         # rol(a,5)
 234         pslld   \$30,$t1
 235         paddd   $t0,$e                          # e+=Ch(b,c,d)
 236
 237         psrld   \$2,$b
 238         paddd   $t2,$e                          # e+=rol(a,5)
 239          por    $tx,@Xi[1]                      # rol   \$1,@Xi[1]
 240         por     $t1,$b                          # b=rol(b,30)
 241 ___
 242 push(@Xi,shift(@Xi));
 243 }
 244
 245 sub BODY_20_39 {
 246 my ($i,$a,$b,$c,$d,$e)=@_;
 247 my $j=$i+1;
 248
 249 $code.=<<___ if ($i<79);
 250         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 251         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 252
 253         movdqa  $a,$t2
 254         movdqa  $d,$t0
 255          pxor   `&Xi_off($j+8)`,@Xi[1]
 256         paddd   $K,$e                           # e+=K_20_39
 257         pslld   \$5,$t2
 258         pxor    $b,$t0
 259
 260         movdqa  $a,$t3
 261 ___
 262 $code.=<<___ if ($i<72);
 263         movdqa  @Xi[0],`&Xi_off($i)`
 264 ___
 265 $code.=<<___ if ($i<79);
 266         paddd   @Xi[0],$e                       # e+=X[i]
 267          pxor   @Xi[3],@Xi[1]
 268         psrld   \$27,$t3
 269         pxor    $c,$t0                          # Parity(b,c,d)
 270         movdqa  $b,$t1
 271
 272         pslld   \$30,$t1
 273          movdqa @Xi[1],$tx
 274         por     $t3,$t2                         # rol(a,5)
 275          psrld  \$31,$tx
 276         paddd   $t0,$e                          # e+=Parity(b,c,d)
 277          paddd  @Xi[1],@Xi[1]
 278
 279         psrld   \$2,$b
 280         paddd   $t2,$e                          # e+=rol(a,5)
 281          por    $tx,@Xi[1]                      # rol(@Xi[1],1)
 282         por     $t1,$b                          # b=rol(b,30)
 283 ___
 284 $code.=<<___ if ($i==79);
 285         movdqa  $a,$t2
 286         paddd   $K,$e                           # e+=K_20_39
 287         movdqa  $d,$t0
 288         pslld   \$5,$t2
 289         pxor    $b,$t0
 290
 291         movdqa  $a,$t3
 292         paddd   @Xi[0],$e                       # e+=X[i]
 293         psrld   \$27,$t3
 294         movdqa  $b,$t1
 295         pxor    $c,$t0                          # Parity(b,c,d)
 296
 297         pslld   \$30,$t1
 298         por     $t3,$t2                         # rol(a,5)
 299         paddd   $t0,$e                          # e+=Parity(b,c,d)
 300
 301         psrld   \$2,$b
 302         paddd   $t2,$e                          # e+=rol(a,5)
 303         por     $t1,$b                          # b=rol(b,30)
 304 ___
 305 push(@Xi,shift(@Xi));
 306 }
 307
 308 sub BODY_40_59 {
 309 my ($i,$a,$b,$c,$d,$e)=@_;
 310 my $j=$i+1;
 311
 312 $code.=<<___;
 313         pxor    @Xi[-2],@Xi[1]                  # "X[13]"
 314         movdqa  `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 315
 316         movdqa  $a,$t2
 317         movdqa  $d,$t1
 318          pxor   `&Xi_off($j+8)`,@Xi[1]
 319         pxor    @Xi[3],@Xi[1]
 320         paddd   $K,$e                           # e+=K_40_59
 321         pslld   \$5,$t2
 322         movdqa  $a,$t3
 323         pand    $c,$t1
 324
 325         movdqa  $d,$t0
 326          movdqa @Xi[1],$tx
 327         psrld   \$27,$t3
 328         paddd   $t1,$e
 329         pxor    $c,$t0
 330
 331         movdqa  @Xi[0],`&Xi_off($i)`
 332         paddd   @Xi[0],$e                       # e+=X[i]
 333         por     $t3,$t2                         # rol(a,5)
 334          psrld  \$31,$tx
 335         pand    $b,$t0
 336         movdqa  $b,$t1
 337
 338         pslld   \$30,$t1
 339          paddd  @Xi[1],@Xi[1]
 340         paddd   $t0,$e                          # e+=Maj(b,d,c)
 341
 342         psrld   \$2,$b
 343         paddd   $t2,$e                          # e+=rol(a,5)
 344          por    $tx,@Xi[1]                      # rol(@X[1],1)
 345         por     $t1,$b                          # b=rol(b,30)
 346 ___
 347 push(@Xi,shift(@Xi));
 348 }
 349
 350 $code.=<<___;
 351 .text
 352
 353 .extern OPENSSL_ia32cap_P
 354
 355 .globl  sha1_multi_block
 356 .type   sha1_multi_block,\@function,3
 357 .align  32
 358 sha1_multi_block:
 359         mov     OPENSSL_ia32cap_P+4(%rip),%rcx
 360         bt      \$61,%rcx                       # check SHA bit
 361         jc      _shaext_shortcut
 362 ___
 363 $code.=<<___ if ($avx);
 364         test    \$`1<<28`,%ecx
 365         jnz     _avx_shortcut
 366 ___
 367 $code.=<<___;
 368         mov     %rsp,%rax
 369         push    %rbx
 370         push    %rbp
 371 ___
 372 $code.=<<___ if ($win64);
 373         lea     -0xa8(%rsp),%rsp
 374         movaps  %xmm6,(%rsp)
 375         movaps  %xmm7,0x10(%rsp)
 376         movaps  %xmm8,0x20(%rsp)
 377         movaps  %xmm9,0x30(%rsp)
 378         movaps  %xmm10,-0x78(%rax)
 379         movaps  %xmm11,-0x68(%rax)
 380         movaps  %xmm12,-0x58(%rax)
 381         movaps  %xmm13,-0x48(%rax)
 382         movaps  %xmm14,-0x38(%rax)
 383         movaps  %xmm15,-0x28(%rax)
 384 ___
 385 $code.=<<___;
 386         sub     \$`$REG_SZ*18`,%rsp
 387         and     \$-256,%rsp
 388         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 389 .Lbody:
 390         lea     K_XX_XX(%rip),$Tbl
 391         lea     `$REG_SZ*16`(%rsp),%rbx
 392
 393 .Loop_grande:
 394         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
 395         xor     $num,$num
 396 ___
 397 for($i=0;$i<4;$i++) {
 398     $code.=<<___;
 399         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 400         mov     `16*$i+8`($inp),%ecx            # number of blocks
 401         cmp     $num,%ecx
 402         cmovg   %ecx,$num                       # find maximum
 403         test    %ecx,%ecx
 404         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 405         cmovle  $Tbl,@ptr[$i]                   # cancel input
 406 ___
 407 }
 408 $code.=<<___;
 409         test    $num,$num
 410         jz      .Ldone
 411
 412         movdqu  0x00($ctx),$A                   # load context
 413          lea    128(%rsp),%rax
 414         movdqu  0x20($ctx),$B
 415         movdqu  0x40($ctx),$C
 416         movdqu  0x60($ctx),$D
 417         movdqu  0x80($ctx),$E
 418         movdqa  0x60($Tbl),$tx                  # pbswap_mask
 419         movdqa  -0x20($Tbl),$K                  # K_00_19
 420         jmp     .Loop
 421
 422 .align  32
 423 .Loop:
 424 ___
 425 for($i=0;$i<20;$i++)    { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
 426 $code.="        movdqa  0x00($Tbl),$K\n";       # K_20_39
 427 for(;$i<40;$i++)        { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
 428 $code.="        movdqa  0x20($Tbl),$K\n";       # K_40_59
 429 for(;$i<60;$i++)        { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
 430 $code.="        movdqa  0x40($Tbl),$K\n";       # K_60_79
 431 for(;$i<80;$i++)        { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
 432 $code.=<<___;
 433         movdqa  (%rbx),@Xi[0]                   # pull counters
 434         mov     \$1,%ecx
 435         cmp     4*0(%rbx),%ecx                  # examinte counters
 436         pxor    $t2,$t2
 437         cmovge  $Tbl,@ptr[0]                    # cancel input
 438         cmp     4*1(%rbx),%ecx
 439         movdqa  @Xi[0],@Xi[1]
 440         cmovge  $Tbl,@ptr[1]
 441         cmp     4*2(%rbx),%ecx
 442         pcmpgtd $t2,@Xi[1]                      # mask value
 443         cmovge  $Tbl,@ptr[2]
 444         cmp     4*3(%rbx),%ecx
 445         paddd   @Xi[1],@Xi[0]                   # counters--
 446         cmovge  $Tbl,@ptr[3]
 447
 448         movdqu  0x00($ctx),$t0
 449         pand    @Xi[1],$A
 450         movdqu  0x20($ctx),$t1
 451         pand    @Xi[1],$B
 452         paddd   $t0,$A
 453         movdqu  0x40($ctx),$t2
 454         pand    @Xi[1],$C
 455         paddd   $t1,$B
 456         movdqu  0x60($ctx),$t3
 457         pand    @Xi[1],$D
 458         paddd   $t2,$C
 459         movdqu  0x80($ctx),$tx
 460         pand    @Xi[1],$E
 461         movdqu  $A,0x00($ctx)
 462         paddd   $t3,$D
 463         movdqu  $B,0x20($ctx)
 464         paddd   $tx,$E
 465         movdqu  $C,0x40($ctx)
 466         movdqu  $D,0x60($ctx)
 467         movdqu  $E,0x80($ctx)
 468
 469         movdqa  @Xi[0],(%rbx)                   # save counters
 470         movdqa  0x60($Tbl),$tx                  # pbswap_mask
 471         movdqa  -0x20($Tbl),$K                  # K_00_19
 472         dec     $num
 473         jnz     .Loop
 474
 475         mov     `$REG_SZ*17+8`(%rsp),$num
 476         lea     $REG_SZ($ctx),$ctx
 477         lea     `16*$REG_SZ/4`($inp),$inp
 478         dec     $num
 479         jnz     .Loop_grande
 480
 481 .Ldone:
 482         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
 483 ___
 484 $code.=<<___ if ($win64);
 485         movaps  -0xb8(%rax),%xmm6
 486         movaps  -0xa8(%rax),%xmm7
 487         movaps  -0x98(%rax),%xmm8
 488         movaps  -0x88(%rax),%xmm9
 489         movaps  -0x78(%rax),%xmm10
 490         movaps  -0x68(%rax),%xmm11
 491         movaps  -0x58(%rax),%xmm12
 492         movaps  -0x48(%rax),%xmm13
 493         movaps  -0x38(%rax),%xmm14
 494         movaps  -0x28(%rax),%xmm15
 495 ___
 496 $code.=<<___;
 497         mov     -16(%rax),%rbp
 498         mov     -8(%rax),%rbx
 499         lea     (%rax),%rsp
 500 .Lepilogue:
 501         ret
 502 .size   sha1_multi_block,.-sha1_multi_block
 503 ___
 504                                                 {{{
 505 my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10));
 506 my @MSG0=map("%xmm$_",(4..7));
 507 my @MSG1=map("%xmm$_",(11..14));
 508
 509 $code.=<<___;
 510 .type   sha1_multi_block_shaext,\@function,3
 511 .align  32
 512 sha1_multi_block_shaext:
 513 _shaext_shortcut:
 514         mov     %rsp,%rax
 515         push    %rbx
 516         push    %rbp
 517 ___
 518 $code.=<<___ if ($win64);
 519         lea     -0xa8(%rsp),%rsp
 520         movaps  %xmm6,(%rsp)
 521         movaps  %xmm7,0x10(%rsp)
 522         movaps  %xmm8,0x20(%rsp)
 523         movaps  %xmm9,0x30(%rsp)
 524         movaps  %xmm10,-0x78(%rax)
 525         movaps  %xmm11,-0x68(%rax)
 526         movaps  %xmm12,-0x58(%rax)
 527         movaps  %xmm13,-0x48(%rax)
 528         movaps  %xmm14,-0x38(%rax)
 529         movaps  %xmm15,-0x28(%rax)
 530 ___
 531 $code.=<<___;
 532         sub     \$`$REG_SZ*18`,%rsp
 533         shl     \$1,$num                        # we process pair at a time
 534         and     \$-256,%rsp
 535         lea     0x40($ctx),$ctx                 # size optimization
 536         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 537 .Lbody_shaext:
 538         lea     `$REG_SZ*16`(%rsp),%rbx
 539         movdqa  K_XX_XX+0x80(%rip),$BSWAP       # byte-n-word swap
 540
 541 .Loop_grande_shaext:
 542         mov     $num,`$REG_SZ*17+8`(%rsp)       # orignal $num
 543         xor     $num,$num
 544 ___
 545 for($i=0;$i<2;$i++) {
 546     $code.=<<___;
 547         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 548         mov     `16*$i+8`($inp),%ecx            # number of blocks
 549         cmp     $num,%ecx
 550         cmovg   %ecx,$num                       # find maximum
 551         test    %ecx,%ecx
 552         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 553         cmovle  %rsp,@ptr[$i]                   # cancel input
 554 ___
 555 }
 556 $code.=<<___;
 557         test    $num,$num
 558         jz      .Ldone_shaext
 559
 560         movq            0x00-0x40($ctx),$ABCD0  # a1.a0
 561         movq            0x20-0x40($ctx),@MSG0[0]# b1.b0
 562         movq            0x40-0x40($ctx),@MSG0[1]# c1.c0
 563         movq            0x60-0x40($ctx),@MSG0[2]# d1.d0
 564         movq            0x80-0x40($ctx),@MSG0[3]# e1.e0
 565
 566         punpckldq       @MSG0[0],$ABCD0         # b1.a1.b0.a0
 567         punpckldq       @MSG0[2],@MSG0[1]       # d1.c1.d0.c0
 568
 569         movdqa          $ABCD0,$ABCD1
 570         punpcklqdq      @MSG0[1],$ABCD0         # d0.c0.b0.a0
 571         punpckhqdq      @MSG0[1],$ABCD1         # d1.c1.b1.a1
 572
 573         pshufd          \$0b00111111,@MSG0[3],$E0
 574         pshufd          \$0b01111111,@MSG0[3],$E1
 575         pshufd          \$0b00011011,$ABCD0,$ABCD0
 576         pshufd          \$0b00011011,$ABCD1,$ABCD1
 577         jmp             .Loop_shaext
 578
 579 .align  32
 580 .Loop_shaext:
 581         movdqu          0x00(@ptr[0]),@MSG0[0]
 582          movdqu         0x00(@ptr[1]),@MSG1[0]
 583         movdqu          0x10(@ptr[0]),@MSG0[1]
 584          movdqu         0x10(@ptr[1]),@MSG1[1]
 585         movdqu          0x20(@ptr[0]),@MSG0[2]
 586         pshufb          $BSWAP,@MSG0[0]
 587          movdqu         0x20(@ptr[1]),@MSG1[2]
 588          pshufb         $BSWAP,@MSG1[0]
 589         movdqu          0x30(@ptr[0]),@MSG0[3]
 590         lea             0x40(@ptr[0]),@ptr[0]
 591         pshufb          $BSWAP,@MSG0[1]
 592          movdqu         0x30(@ptr[1]),@MSG1[3]
 593          lea            0x40(@ptr[1]),@ptr[1]
 594          pshufb         $BSWAP,@MSG1[1]
 595
 596         movdqa          $E0,0x50(%rsp)          # offload
 597         paddd           @MSG0[0],$E0
 598          movdqa         $E1,0x70(%rsp)
 599          paddd          @MSG1[0],$E1
 600         movdqa          $ABCD0,0x40(%rsp)       # offload
 601         movdqa          $ABCD0,$E0_
 602          movdqa         $ABCD1,0x60(%rsp)
 603          movdqa         $ABCD1,$E1_
 604         sha1rnds4       \$0,$E0,$ABCD0          # 0-3
 605         sha1nexte       @MSG0[1],$E0_
 606          sha1rnds4      \$0,$E1,$ABCD1          # 0-3
 607          sha1nexte      @MSG1[1],$E1_
 608         pshufb          $BSWAP,@MSG0[2]
 609         prefetcht0      127(@ptr[0])
 610         sha1msg1        @MSG0[1],@MSG0[0]
 611          pshufb         $BSWAP,@MSG1[2]
 612          prefetcht0     127(@ptr[1])
 613          sha1msg1       @MSG1[1],@MSG1[0]
 614
 615         pshufb          $BSWAP,@MSG0[3]
 616         movdqa          $ABCD0,$E0
 617          pshufb         $BSWAP,@MSG1[3]
 618          movdqa         $ABCD1,$E1
 619         sha1rnds4       \$0,$E0_,$ABCD0         # 4-7
 620         sha1nexte       @MSG0[2],$E0
 621          sha1rnds4      \$0,$E1_,$ABCD1         # 4-7
 622          sha1nexte      @MSG1[2],$E1
 623         pxor            @MSG0[2],@MSG0[0]
 624         sha1msg1        @MSG0[2],@MSG0[1]
 625          pxor           @MSG1[2],@MSG1[0]
 626          sha1msg1       @MSG1[2],@MSG1[1]
 627 ___
 628 for($i=2;$i<20-4;$i++) {
 629 $code.=<<___;
 630         movdqa          $ABCD0,$E0_
 631          movdqa         $ABCD1,$E1_
 632         sha1rnds4       \$`int($i/5)`,$E0,$ABCD0        # 8-11
 633         sha1nexte       @MSG0[3],$E0_
 634          sha1rnds4      \$`int($i/5)`,$E1,$ABCD1        # 8-11
 635          sha1nexte      @MSG1[3],$E1_
 636         sha1msg2        @MSG0[3],@MSG0[0]
 637          sha1msg2       @MSG1[3],@MSG1[0]
 638         pxor            @MSG0[3],@MSG0[1]
 639         sha1msg1        @MSG0[3],@MSG0[2]
 640          pxor           @MSG1[3],@MSG1[1]
 641          sha1msg1       @MSG1[3],@MSG1[2]
 642 ___
 643         ($E0,$E0_)=($E0_,$E0);          ($E1,$E1_)=($E1_,$E1);
 644         push(@MSG0,shift(@MSG0));       push(@MSG1,shift(@MSG1));
 645 }
 646 $code.=<<___;
 647         movdqa          $ABCD0,$E0_
 648          movdqa         $ABCD1,$E1_
 649         sha1rnds4       \$3,$E0,$ABCD0          # 64-67
 650         sha1nexte       @MSG0[3],$E0_
 651          sha1rnds4      \$3,$E1,$ABCD1          # 64-67
 652          sha1nexte      @MSG1[3],$E1_
 653         sha1msg2        @MSG0[3],@MSG0[0]
 654          sha1msg2       @MSG1[3],@MSG1[0]
 655         pxor            @MSG0[3],@MSG0[1]
 656          pxor           @MSG1[3],@MSG1[1]
 657
 658         mov             \$1,%ecx
 659         pxor            @MSG0[2],@MSG0[2]       # zero
 660         cmp             4*0(%rbx),%ecx          # examine counters
 661         cmovge          %rsp,@ptr[0]            # cancel input
 662
 663         movdqa          $ABCD0,$E0
 664          movdqa         $ABCD1,$E1
 665         sha1rnds4       \$3,$E0_,$ABCD0         # 68-71
 666         sha1nexte       @MSG0[0],$E0
 667          sha1rnds4      \$3,$E1_,$ABCD1         # 68-71
 668          sha1nexte      @MSG1[0],$E1
 669         sha1msg2        @MSG0[0],@MSG0[1]
 670          sha1msg2       @MSG1[0],@MSG1[1]
 671
 672         cmp             4*1(%rbx),%ecx
 673         cmovge          %rsp,@ptr[1]
 674         movq            (%rbx),@MSG0[0]         # pull counters
 675
 676         movdqa          $ABCD0,$E0_
 677          movdqa         $ABCD1,$E1_
 678         sha1rnds4       \$3,$E0,$ABCD0          # 72-75
 679         sha1nexte       @MSG0[1],$E0_
 680          sha1rnds4      \$3,$E1,$ABCD1          # 72-75
 681          sha1nexte      @MSG1[1],$E1_
 682
 683         pshufd          \$0x00,@MSG0[0],@MSG1[2]
 684         pshufd          \$0x55,@MSG0[0],@MSG1[3]
 685         movdqa          @MSG0[0],@MSG0[1]
 686         pcmpgtd         @MSG0[2],@MSG1[2]
 687         pcmpgtd         @MSG0[2],@MSG1[3]
 688
 689         movdqa          $ABCD0,$E0
 690          movdqa         $ABCD1,$E1
 691         sha1rnds4       \$3,$E0_,$ABCD0         # 76-79
 692         sha1nexte       $MSG0[2],$E0
 693          sha1rnds4      \$3,$E1_,$ABCD1         # 76-79
 694          sha1nexte      $MSG0[2],$E1
 695
 696         pcmpgtd         @MSG0[2],@MSG0[1]       # counter mask
 697         pand            @MSG1[2],$ABCD0
 698         pand            @MSG1[2],$E0
 699          pand           @MSG1[3],$ABCD1
 700          pand           @MSG1[3],$E1
 701         paddd           @MSG0[1],@MSG0[0]       # counters--
 702
 703         paddd           0x40(%rsp),$ABCD0
 704         paddd           0x50(%rsp),$E0
 705          paddd          0x60(%rsp),$ABCD1
 706          paddd          0x70(%rsp),$E1
 707
 708         movq            @MSG0[0],(%rbx)         # save counters
 709         dec             $num
 710         jnz             .Loop_shaext
 711
 712         mov             `$REG_SZ*17+8`(%rsp),$num
 713
 714         pshufd          \$0b00011011,$ABCD0,$ABCD0
 715         pshufd          \$0b00011011,$ABCD1,$ABCD1
 716
 717         movdqa          $ABCD0,@MSG0[0]
 718         punpckldq       $ABCD1,$ABCD0           # b1.b0.a1.a0
 719         punpckhdq       $ABCD1,@MSG0[0]         # d1.d0.c1.c0
 720         punpckhdq       $E1,$E0                 # e1.e0.xx.xx
 721         movq            $ABCD0,0x00-0x40($ctx)  # a1.a0
 722         psrldq          \$8,$ABCD0
 723         movq            @MSG0[0],0x40-0x40($ctx)# c1.c0
 724         psrldq          \$8,@MSG0[0]
 725         movq            $ABCD0,0x20-0x40($ctx)  # b1.b0
 726         psrldq          \$8,$E0
 727         movq            @MSG0[0],0x60-0x40($ctx)# d1.d0
 728         movq            $E0,0x80-0x40($ctx)     # e1.e0
 729
 730         lea     `$REG_SZ/2`($ctx),$ctx
 731         lea     `16*2`($inp),$inp
 732         dec     $num
 733         jnz     .Loop_grande_shaext
 734
 735 .Ldone_shaext:
 736         #mov    `$REG_SZ*17`(%rsp),%rax         # original %rsp
 737 ___
 738 $code.=<<___ if ($win64);
 739         movaps  -0xb8(%rax),%xmm6
 740         movaps  -0xa8(%rax),%xmm7
 741         movaps  -0x98(%rax),%xmm8
 742         movaps  -0x88(%rax),%xmm9
 743         movaps  -0x78(%rax),%xmm10
 744         movaps  -0x68(%rax),%xmm11
 745         movaps  -0x58(%rax),%xmm12
 746         movaps  -0x48(%rax),%xmm13
 747         movaps  -0x38(%rax),%xmm14
 748         movaps  -0x28(%rax),%xmm15
 749 ___
 750 $code.=<<___;
 751         mov     -16(%rax),%rbp
 752         mov     -8(%rax),%rbx
 753         lea     (%rax),%rsp
 754 .Lepilogue_shaext:
 755         ret
 756 .size   sha1_multi_block_shaext,.-sha1_multi_block_shaext
 757 ___
 758                                                 }}}
 759
 760                                                 if ($avx) {{{
 761 sub BODY_00_19_avx {
 762 my ($i,$a,$b,$c,$d,$e)=@_;
 763 my $j=$i+1;
 764 my $k=$i+2;
 765 my $vpack = $REG_SZ==16 ? "vpunpckldq" : "vinserti128";
 766 my $ptr_n = $REG_SZ==16 ? @ptr[1] : @ptr[4];
 767
 768 $code.=<<___ if ($i==0 && $REG_SZ==16);
 769         vmovd           (@ptr[0]),@Xi[0]
 770          lea            `16*4`(@ptr[0]),@ptr[0]
 771         vmovd           (@ptr[1]),@Xi[2]        # borrow Xi[2]
 772          lea            `16*4`(@ptr[1]),@ptr[1]
 773         vpinsrd         \$1,(@ptr[2]),@Xi[0],@Xi[0]
 774          lea            `16*4`(@ptr[2]),@ptr[2]
 775         vpinsrd         \$1,(@ptr[3]),@Xi[2],@Xi[2]
 776          lea            `16*4`(@ptr[3]),@ptr[3]
 777          vmovd          `4*$j-16*4`(@ptr[0]),@Xi[1]
 778         vpunpckldq      @Xi[2],@Xi[0],@Xi[0]
 779          vmovd          `4*$j-16*4`($ptr_n),$t3
 780         vpshufb         $tx,@Xi[0],@Xi[0]
 781 ___
 782 $code.=<<___ if ($i<15 && $REG_SZ==16);         # just load input
 783          vpinsrd        \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
 784          vpinsrd        \$1,`4*$j-16*4`(@ptr[3]),$t3,$t3
 785 ___
 786 $code.=<<___ if ($i==0 && $REG_SZ==32);
 787         vmovd           (@ptr[0]),@Xi[0]
 788          lea            `16*4`(@ptr[0]),@ptr[0]
 789         vmovd           (@ptr[4]),@Xi[2]        # borrow Xi[2]
 790          lea            `16*4`(@ptr[4]),@ptr[4]
 791         vmovd           (@ptr[1]),$t2
 792          lea            `16*4`(@ptr[1]),@ptr[1]
 793         vmovd           (@ptr[5]),$t1
 794          lea            `16*4`(@ptr[5]),@ptr[5]
 795         vpinsrd         \$1,(@ptr[2]),@Xi[0],@Xi[0]
 796          lea            `16*4`(@ptr[2]),@ptr[2]
 797         vpinsrd         \$1,(@ptr[6]),@Xi[2],@Xi[2]
 798          lea            `16*4`(@ptr[6]),@ptr[6]
 799         vpinsrd         \$1,(@ptr[3]),$t2,$t2
 800          lea            `16*4`(@ptr[3]),@ptr[3]
 801         vpunpckldq      $t2,@Xi[0],@Xi[0]
 802         vpinsrd         \$1,(@ptr[7]),$t1,$t1
 803          lea            `16*4`(@ptr[7]),@ptr[7]
 804         vpunpckldq      $t1,@Xi[2],@Xi[2]
 805          vmovd          `4*$j-16*4`(@ptr[0]),@Xi[1]
 806         vinserti128     @Xi[2],@Xi[0],@Xi[0]
 807          vmovd          `4*$j-16*4`($ptr_n),$t3
 808         vpshufb         $tx,@Xi[0],@Xi[0]
 809 ___
 810 $code.=<<___ if ($i<15 && $REG_SZ==32);         # just load input
 811          vmovd          `4*$j-16*4`(@ptr[1]),$t2
 812          vmovd          `4*$j-16*4`(@ptr[5]),$t1
 813          vpinsrd        \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
 814          vpinsrd        \$1,`4*$j-16*4`(@ptr[6]),$t3,$t3
 815          vpinsrd        \$1,`4*$j-16*4`(@ptr[3]),$t2,$t2
 816          vpunpckldq     $t2,@Xi[1],@Xi[1]
 817          vpinsrd        \$1,`4*$j-16*4`(@ptr[7]),$t1,$t1
 818          vpunpckldq     $t1,$t3,$t3
 819 ___
 820 $code.=<<___ if ($i<14);
 821         vpaddd  $K,$e,$e                        # e+=K_00_19
 822         vpslld  \$5,$a,$t2
 823         vpandn  $d,$b,$t1
 824         vpand   $c,$b,$t0
 825
 826         vmovdqa @Xi[0],`&Xi_off($i)`
 827         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 828          $vpack         $t3,@Xi[1],@Xi[1]
 829         vpsrld  \$27,$a,$t3
 830         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 831          vmovd          `4*$k-16*4`(@ptr[0]),@Xi[2]
 832
 833         vpslld  \$30,$b,$t1
 834         vpor    $t3,$t2,$t2                     # rol(a,5)
 835          vmovd          `4*$k-16*4`($ptr_n),$t3
 836         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 837
 838         vpsrld  \$2,$b,$b
 839         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 840          vpshufb        $tx,@Xi[1],@Xi[1]
 841         vpor    $t1,$b,$b                       # b=rol(b,30)
 842 ___
 843 $code.=<<___ if ($i==14);
 844         vpaddd  $K,$e,$e                        # e+=K_00_19
 845          prefetcht0     63(@ptr[0])
 846         vpslld  \$5,$a,$t2
 847         vpandn  $d,$b,$t1
 848         vpand   $c,$b,$t0
 849
 850         vmovdqa @Xi[0],`&Xi_off($i)`
 851         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 852          $vpack         $t3,@Xi[1],@Xi[1]
 853         vpsrld  \$27,$a,$t3
 854          prefetcht0     63(@ptr[1])
 855         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 856
 857         vpslld  \$30,$b,$t1
 858         vpor    $t3,$t2,$t2                     # rol(a,5)
 859          prefetcht0     63(@ptr[2])
 860         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 861
 862         vpsrld  \$2,$b,$b
 863         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 864          prefetcht0     63(@ptr[3])
 865          vpshufb        $tx,@Xi[1],@Xi[1]
 866         vpor    $t1,$b,$b                       # b=rol(b,30)
 867 ___
 868 $code.=<<___ if ($i>=13 && $i<15);
 869         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # preload "X[2]"
 870 ___
 871 $code.=<<___ if ($i>=15);                       # apply Xupdate
 872         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 873         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 874
 875         vpaddd  $K,$e,$e                        # e+=K_00_19
 876         vpslld  \$5,$a,$t2
 877         vpandn  $d,$b,$t1
 878          `"prefetcht0   63(@ptr[4])"            if ($i==15 && $REG_SZ==32)`
 879         vpand   $c,$b,$t0
 880
 881         vmovdqa @Xi[0],`&Xi_off($i)`
 882         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 883          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 884         vpsrld  \$27,$a,$t3
 885         vpxor   $t1,$t0,$t0                     # Ch(b,c,d)
 886          vpxor  @Xi[3],@Xi[1],@Xi[1]
 887          `"prefetcht0   63(@ptr[5])"            if ($i==15 && $REG_SZ==32)`
 888
 889         vpslld  \$30,$b,$t1
 890         vpor    $t3,$t2,$t2                     # rol(a,5)
 891         vpaddd  $t0,$e,$e                       # e+=Ch(b,c,d)
 892          `"prefetcht0   63(@ptr[6])"            if ($i==15 && $REG_SZ==32)`
 893          vpsrld \$31,@Xi[1],$tx
 894          vpaddd @Xi[1],@Xi[1],@Xi[1]
 895
 896         vpsrld  \$2,$b,$b
 897          `"prefetcht0   63(@ptr[7])"            if ($i==15 && $REG_SZ==32)`
 898         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 899          vpor   $tx,@Xi[1],@Xi[1]               # rol   \$1,@Xi[1]
 900         vpor    $t1,$b,$b                       # b=rol(b,30)
 901 ___
 902 push(@Xi,shift(@Xi));
 903 }
 904
 905 sub BODY_20_39_avx {
 906 my ($i,$a,$b,$c,$d,$e)=@_;
 907 my $j=$i+1;
 908
 909 $code.=<<___ if ($i<79);
 910         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 911         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 912
 913         vpslld  \$5,$a,$t2
 914         vpaddd  $K,$e,$e                        # e+=K_20_39
 915         vpxor   $b,$d,$t0
 916 ___
 917 $code.=<<___ if ($i<72);
 918         vmovdqa @Xi[0],`&Xi_off($i)`
 919 ___
 920 $code.=<<___ if ($i<79);
 921         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 922          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 923         vpsrld  \$27,$a,$t3
 924         vpxor   $c,$t0,$t0                      # Parity(b,c,d)
 925          vpxor  @Xi[3],@Xi[1],@Xi[1]
 926
 927         vpslld  \$30,$b,$t1
 928         vpor    $t3,$t2,$t2                     # rol(a,5)
 929         vpaddd  $t0,$e,$e                       # e+=Parity(b,c,d)
 930          vpsrld \$31,@Xi[1],$tx
 931          vpaddd @Xi[1],@Xi[1],@Xi[1]
 932
 933         vpsrld  \$2,$b,$b
 934         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 935          vpor   $tx,@Xi[1],@Xi[1]               # rol(@Xi[1],1)
 936         vpor    $t1,$b,$b                       # b=rol(b,30)
 937 ___
 938 $code.=<<___ if ($i==79);
 939         vpslld  \$5,$a,$t2
 940         vpaddd  $K,$e,$e                        # e+=K_20_39
 941         vpxor   $b,$d,$t0
 942
 943         vpsrld  \$27,$a,$t3
 944         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 945         vpxor   $c,$t0,$t0                      # Parity(b,c,d)
 946
 947         vpslld  \$30,$b,$t1
 948         vpor    $t3,$t2,$t2                     # rol(a,5)
 949         vpaddd  $t0,$e,$e                       # e+=Parity(b,c,d)
 950
 951         vpsrld  \$2,$b,$b
 952         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 953         vpor    $t1,$b,$b                       # b=rol(b,30)
 954 ___
 955 push(@Xi,shift(@Xi));
 956 }
 957
 958 sub BODY_40_59_avx {
 959 my ($i,$a,$b,$c,$d,$e)=@_;
 960 my $j=$i+1;
 961
 962 $code.=<<___;
 963         vpxor   @Xi[-2],@Xi[1],@Xi[1]           # "X[13]"
 964         vmovdqa `&Xi_off($j+2)`,@Xi[3]          # "X[2]"
 965
 966         vpaddd  $K,$e,$e                        # e+=K_40_59
 967         vpslld  \$5,$a,$t2
 968         vpand   $c,$d,$t1
 969          vpxor  `&Xi_off($j+8)`,@Xi[1],@Xi[1]
 970
 971         vpaddd  $t1,$e,$e
 972         vpsrld  \$27,$a,$t3
 973         vpxor   $c,$d,$t0
 974          vpxor  @Xi[3],@Xi[1],@Xi[1]
 975
 976         vmovdqu @Xi[0],`&Xi_off($i)`
 977         vpaddd  @Xi[0],$e,$e                    # e+=X[i]
 978         vpor    $t3,$t2,$t2                     # rol(a,5)
 979          vpsrld \$31,@Xi[1],$tx
 980         vpand   $b,$t0,$t0
 981          vpaddd @Xi[1],@Xi[1],@Xi[1]
 982
 983         vpslld  \$30,$b,$t1
 984         vpaddd  $t0,$e,$e                       # e+=Maj(b,d,c)
 985
 986         vpsrld  \$2,$b,$b
 987         vpaddd  $t2,$e,$e                       # e+=rol(a,5)
 988          vpor   $tx,@Xi[1],@Xi[1]               # rol(@X[1],1)
 989         vpor    $t1,$b,$b                       # b=rol(b,30)
 990 ___
 991 push(@Xi,shift(@Xi));
 992 }
 993
 994 $code.=<<___;
 995 .type   sha1_multi_block_avx,\@function,3
 996 .align  32
 997 sha1_multi_block_avx:
 998 _avx_shortcut:
 999 ___
1000 $code.=<<___ if ($avx>1);
1001         shr     \$32,%rcx
1002         cmp     \$2,$num
1003         jb      .Lavx
1004         test    \$`1<<5`,%ecx
1005         jnz     _avx2_shortcut
1006         jmp     .Lavx
1007 .align  32
1008 .Lavx:
1009 ___
1010 $code.=<<___;
1011         mov     %rsp,%rax
1012         push    %rbx
1013         push    %rbp
1014 ___
1015 $code.=<<___ if ($win64);
1016         lea     -0xa8(%rsp),%rsp
1017         movaps  %xmm6,(%rsp)
1018         movaps  %xmm7,0x10(%rsp)
1019         movaps  %xmm8,0x20(%rsp)
1020         movaps  %xmm9,0x30(%rsp)
1021         movaps  %xmm10,-0x78(%rax)
1022         movaps  %xmm11,-0x68(%rax)
1023         movaps  %xmm12,-0x58(%rax)
1024         movaps  %xmm13,-0x48(%rax)
1025         movaps  %xmm14,-0x38(%rax)
1026         movaps  %xmm15,-0x28(%rax)
1027 ___
1028 $code.=<<___;
1029         sub     \$`$REG_SZ*18`, %rsp
1030         and     \$-256,%rsp
1031         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1032 .Lbody_avx:
1033         lea     K_XX_XX(%rip),$Tbl
1034         lea     `$REG_SZ*16`(%rsp),%rbx
1035
1036         vzeroupper
1037 .Loop_grande_avx:
1038         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1039         xor     $num,$num
1040 ___
1041 for($i=0;$i<4;$i++) {
1042     $code.=<<___;
1043         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1044         mov     `16*$i+8`($inp),%ecx            # number of blocks
1045         cmp     $num,%ecx
1046         cmovg   %ecx,$num                       # find maximum
1047         test    %ecx,%ecx
1048         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1049         cmovle  $Tbl,@ptr[$i]                   # cancel input
1050 ___
1051 }
1052 $code.=<<___;
1053         test    $num,$num
1054         jz      .Ldone_avx
1055
1056         vmovdqu 0x00($ctx),$A                   # load context
1057          lea    128(%rsp),%rax
1058         vmovdqu 0x20($ctx),$B
1059         vmovdqu 0x40($ctx),$C
1060         vmovdqu 0x60($ctx),$D
1061         vmovdqu 0x80($ctx),$E
1062         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1063         jmp     .Loop_avx
1064
1065 .align  32
1066 .Loop_avx:
1067 ___
1068 $code.="        vmovdqa -0x20($Tbl),$K\n";      # K_00_19
1069 for($i=0;$i<20;$i++)    { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1070 $code.="        vmovdqa 0x00($Tbl),$K\n";       # K_20_39
1071 for(;$i<40;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1072 $code.="        vmovdqa 0x20($Tbl),$K\n";       # K_40_59
1073 for(;$i<60;$i++)        { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1074 $code.="        vmovdqa 0x40($Tbl),$K\n";       # K_60_79
1075 for(;$i<80;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1076 $code.=<<___;
1077         mov     \$1,%ecx
1078 ___
1079 for($i=0;$i<4;$i++) {
1080     $code.=<<___;
1081         cmp     `4*$i`(%rbx),%ecx               # examine counters
1082         cmovge  $Tbl,@ptr[$i]                   # cancel input
1083 ___
1084 }
1085 $code.=<<___;
1086         vmovdqu (%rbx),$t0                      # pull counters
1087         vpxor   $t2,$t2,$t2
1088         vmovdqa $t0,$t1
1089         vpcmpgtd $t2,$t1,$t1                    # mask value
1090         vpaddd  $t1,$t0,$t0                     # counters--
1091
1092         vpand   $t1,$A,$A
1093         vpand   $t1,$B,$B
1094         vpaddd  0x00($ctx),$A,$A
1095         vpand   $t1,$C,$C
1096         vpaddd  0x20($ctx),$B,$B
1097         vpand   $t1,$D,$D
1098         vpaddd  0x40($ctx),$C,$C
1099         vpand   $t1,$E,$E
1100         vpaddd  0x60($ctx),$D,$D
1101         vpaddd  0x80($ctx),$E,$E
1102         vmovdqu $A,0x00($ctx)
1103         vmovdqu $B,0x20($ctx)
1104         vmovdqu $C,0x40($ctx)
1105         vmovdqu $D,0x60($ctx)
1106         vmovdqu $E,0x80($ctx)
1107
1108         vmovdqu $t0,(%rbx)                      # save counters
1109         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1110         dec     $num
1111         jnz     .Loop_avx
1112
1113         mov     `$REG_SZ*17+8`(%rsp),$num
1114         lea     $REG_SZ($ctx),$ctx
1115         lea     `16*$REG_SZ/4`($inp),$inp
1116         dec     $num
1117         jnz     .Loop_grande_avx
1118
1119 .Ldone_avx:
1120         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
1121         vzeroupper
1122 ___
1123 $code.=<<___ if ($win64);
1124         movaps  -0xb8(%rax),%xmm6
1125         movaps  -0xa8(%rax),%xmm7
1126         movaps  -0x98(%rax),%xmm8
1127         movaps  -0x88(%rax),%xmm9
1128         movaps  -0x78(%rax),%xmm10
1129         movaps  -0x68(%rax),%xmm11
1130         movaps  -0x58(%rax),%xmm12
1131         movaps  -0x48(%rax),%xmm13
1132         movaps  -0x38(%rax),%xmm14
1133         movaps  -0x28(%rax),%xmm15
1134 ___
1135 $code.=<<___;
1136         mov     -16(%rax),%rbp
1137         mov     -8(%rax),%rbx
1138         lea     (%rax),%rsp
1139 .Lepilogue_avx:
1140         ret
1141 .size   sha1_multi_block_avx,.-sha1_multi_block_avx
1142 ___
1143
1144                                                 if ($avx>1) {
1145 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1146
1147 $REG_SZ=32;
1148
1149 @ptr=map("%r$_",(12..15,8..11));
1150
1151 @V=($A,$B,$C,$D,$E)=map("%ymm$_",(0..4));
1152 ($t0,$t1,$t2,$t3,$tx)=map("%ymm$_",(5..9));
1153 @Xi=map("%ymm$_",(10..14));
1154 $K="%ymm15";
1155
1156 $code.=<<___;
1157 .type   sha1_multi_block_avx2,\@function,3
1158 .align  32
1159 sha1_multi_block_avx2:
1160 _avx2_shortcut:
1161         mov     %rsp,%rax
1162         push    %rbx
1163         push    %rbp
1164         push    %r12
1165         push    %r13
1166         push    %r14
1167         push    %r15
1168 ___
1169 $code.=<<___ if ($win64);
1170         lea     -0xa8(%rsp),%rsp
1171         movaps  %xmm6,(%rsp)
1172         movaps  %xmm7,0x10(%rsp)
1173         movaps  %xmm8,0x20(%rsp)
1174         movaps  %xmm9,0x30(%rsp)
1175         movaps  %xmm10,0x40(%rsp)
1176         movaps  %xmm11,0x50(%rsp)
1177         movaps  %xmm12,-0x78(%rax)
1178         movaps  %xmm13,-0x68(%rax)
1179         movaps  %xmm14,-0x58(%rax)
1180         movaps  %xmm15,-0x48(%rax)
1181 ___
1182 $code.=<<___;
1183         sub     \$`$REG_SZ*18`, %rsp
1184         and     \$-256,%rsp
1185         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1186 .Lbody_avx2:
1187         lea     K_XX_XX(%rip),$Tbl
1188         shr     \$1,$num
1189
1190         vzeroupper
1191 .Loop_grande_avx2:
1192         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1193         xor     $num,$num
1194         lea     `$REG_SZ*16`(%rsp),%rbx
1195 ___
1196 for($i=0;$i<8;$i++) {
1197     $code.=<<___;
1198         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1199         mov     `16*$i+8`($inp),%ecx            # number of blocks
1200         cmp     $num,%ecx
1201         cmovg   %ecx,$num                       # find maximum
1202         test    %ecx,%ecx
1203         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1204         cmovle  $Tbl,@ptr[$i]                   # cancel input
1205 ___
1206 }
1207 $code.=<<___;
1208         vmovdqu 0x00($ctx),$A                   # load context
1209          lea    128(%rsp),%rax
1210         vmovdqu 0x20($ctx),$B
1211          lea    256+128(%rsp),%rbx
1212         vmovdqu 0x40($ctx),$C
1213         vmovdqu 0x60($ctx),$D
1214         vmovdqu 0x80($ctx),$E
1215         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1216         jmp     .Loop_avx2
1217
1218 .align  32
1219 .Loop_avx2:
1220 ___
1221 $code.="        vmovdqa -0x20($Tbl),$K\n";      # K_00_19
1222 for($i=0;$i<20;$i++)    { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1223 $code.="        vmovdqa 0x00($Tbl),$K\n";       # K_20_39
1224 for(;$i<40;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1225 $code.="        vmovdqa 0x20($Tbl),$K\n";       # K_40_59
1226 for(;$i<60;$i++)        { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1227 $code.="        vmovdqa 0x40($Tbl),$K\n";       # K_60_79
1228 for(;$i<80;$i++)        { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1229 $code.=<<___;
1230         mov     \$1,%ecx
1231         lea     `$REG_SZ*16`(%rsp),%rbx
1232 ___
1233 for($i=0;$i<8;$i++) {
1234     $code.=<<___;
1235         cmp     `4*$i`(%rbx),%ecx               # examine counters
1236         cmovge  $Tbl,@ptr[$i]                   # cancel input
1237 ___
1238 }
1239 $code.=<<___;
1240         vmovdqu (%rbx),$t0              # pull counters
1241         vpxor   $t2,$t2,$t2
1242         vmovdqa $t0,$t1
1243         vpcmpgtd $t2,$t1,$t1                    # mask value
1244         vpaddd  $t1,$t0,$t0                     # counters--
1245
1246         vpand   $t1,$A,$A
1247         vpand   $t1,$B,$B
1248         vpaddd  0x00($ctx),$A,$A
1249         vpand   $t1,$C,$C
1250         vpaddd  0x20($ctx),$B,$B
1251         vpand   $t1,$D,$D
1252         vpaddd  0x40($ctx),$C,$C
1253         vpand   $t1,$E,$E
1254         vpaddd  0x60($ctx),$D,$D
1255         vpaddd  0x80($ctx),$E,$E
1256         vmovdqu $A,0x00($ctx)
1257         vmovdqu $B,0x20($ctx)
1258         vmovdqu $C,0x40($ctx)
1259         vmovdqu $D,0x60($ctx)
1260         vmovdqu $E,0x80($ctx)
1261
1262         vmovdqu $t0,(%rbx)                      # save counters
1263         lea     256+128(%rsp),%rbx
1264         vmovdqu 0x60($Tbl),$tx                  # pbswap_mask
1265         dec     $num
1266         jnz     .Loop_avx2
1267
1268         #mov    `$REG_SZ*17+8`(%rsp),$num
1269         #lea    $REG_SZ($ctx),$ctx
1270         #lea    `16*$REG_SZ/4`($inp),$inp
1271         #dec    $num
1272         #jnz    .Loop_grande_avx2
1273
1274 .Ldone_avx2:
1275         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
1276         vzeroupper
1277 ___
1278 $code.=<<___ if ($win64);
1279         movaps  -0xd8(%rax),%xmm6
1280         movaps  -0xc8(%rax),%xmm7
1281         movaps  -0xb8(%rax),%xmm8
1282         movaps  -0xa8(%rax),%xmm9
1283         movaps  -0x98(%rax),%xmm10
1284         movaps  -0x88(%rax),%xmm11
1285         movaps  -0x78(%rax),%xmm12
1286         movaps  -0x68(%rax),%xmm13
1287         movaps  -0x58(%rax),%xmm14
1288         movaps  -0x48(%rax),%xmm15
1289 ___
1290 $code.=<<___;
1291         mov     -48(%rax),%r15
1292         mov     -40(%rax),%r14
1293         mov     -32(%rax),%r13
1294         mov     -24(%rax),%r12
1295         mov     -16(%rax),%rbp
1296         mov     -8(%rax),%rbx
1297         lea     (%rax),%rsp
1298 .Lepilogue_avx2:
1299         ret
1300 .size   sha1_multi_block_avx2,.-sha1_multi_block_avx2
1301 ___
1302                                                 }       }}}
1303 $code.=<<___;
1304
1305 .align  256
1306         .long   0x5a827999,0x5a827999,0x5a827999,0x5a827999     # K_00_19
1307         .long   0x5a827999,0x5a827999,0x5a827999,0x5a827999     # K_00_19
1308 K_XX_XX:
1309         .long   0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1     # K_20_39
1310         .long   0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1     # K_20_39
1311         .long   0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc     # K_40_59
1312         .long   0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc     # K_40_59
1313         .long   0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6     # K_60_79
1314         .long   0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6     # K_60_79
1315         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1316         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1317         .byte   0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0
1318         .asciz  "SHA1 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
1319 ___
1320
1321 if ($win64) {
1322 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1323 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1324 $rec="%rcx";
1325 $frame="%rdx";
1326 $context="%r8";
1327 $disp="%r9";
1328
1329 $code.=<<___;
1330 .extern __imp_RtlVirtualUnwind
1331 .type   se_handler,\@abi-omnipotent
1332 .align  16
1333 se_handler:
1334         push    %rsi
1335         push    %rdi
1336         push    %rbx
1337         push    %rbp
1338         push    %r12
1339         push    %r13
1340         push    %r14
1341         push    %r15
1342         pushfq
1343         sub     \$64,%rsp
1344
1345         mov     120($context),%rax      # pull context->Rax
1346         mov     248($context),%rbx      # pull context->Rip
1347
1348         mov     8($disp),%rsi           # disp->ImageBase
1349         mov     56($disp),%r11          # disp->HandlerData
1350
1351         mov     0(%r11),%r10d           # HandlerData[0]
1352         lea     (%rsi,%r10),%r10        # end of prologue label
1353         cmp     %r10,%rbx               # context->Rip<.Lbody
1354         jb      .Lin_prologue
1355
1356         mov     152($context),%rax      # pull context->Rsp
1357
1358         mov     4(%r11),%r10d           # HandlerData[1]
1359         lea     (%rsi,%r10),%r10        # epilogue label
1360         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1361         jae     .Lin_prologue
1362
1363         mov     `16*17`(%rax),%rax      # pull saved stack pointer
1364
1365         mov     -8(%rax),%rbx
1366         mov     -16(%rax),%rbp
1367         mov     %rbx,144($context)      # restore context->Rbx
1368         mov     %rbp,160($context)      # restore context->Rbp
1369
1370         lea     -24-10*16(%rax),%rsi
1371         lea     512($context),%rdi      # &context.Xmm6
1372         mov     \$20,%ecx
1373         .long   0xa548f3fc              # cld; rep movsq
1374
1375 .Lin_prologue:
1376         mov     8(%rax),%rdi
1377         mov     16(%rax),%rsi
1378         mov     %rax,152($context)      # restore context->Rsp
1379         mov     %rsi,168($context)      # restore context->Rsi
1380         mov     %rdi,176($context)      # restore context->Rdi
1381
1382         mov     40($disp),%rdi          # disp->ContextRecord
1383         mov     $context,%rsi           # context
1384         mov     \$154,%ecx              # sizeof(CONTEXT)
1385         .long   0xa548f3fc              # cld; rep movsq
1386
1387         mov     $disp,%rsi
1388         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1389         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1390         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1391         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1392         mov     40(%rsi),%r10           # disp->ContextRecord
1393         lea     56(%rsi),%r11           # &disp->HandlerData
1394         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1395         mov     %r10,32(%rsp)           # arg5
1396         mov     %r11,40(%rsp)           # arg6
1397         mov     %r12,48(%rsp)           # arg7
1398         mov     %rcx,56(%rsp)           # arg8, (NULL)
1399         call    *__imp_RtlVirtualUnwind(%rip)
1400
1401         mov     \$1,%eax                # ExceptionContinueSearch
1402         add     \$64,%rsp
1403         popfq
1404         pop     %r15
1405         pop     %r14
1406         pop     %r13
1407         pop     %r12
1408         pop     %rbp
1409         pop     %rbx
1410         pop     %rdi
1411         pop     %rsi
1412         ret
1413 .size   se_handler,.-se_handler
1414 ___
1415 $code.=<<___ if ($avx>1);
1416 .type   avx2_handler,\@abi-omnipotent
1417 .align  16
1418 avx2_handler:
1419         push    %rsi
1420         push    %rdi
1421         push    %rbx
1422         push    %rbp
1423         push    %r12
1424         push    %r13
1425         push    %r14
1426         push    %r15
1427         pushfq
1428         sub     \$64,%rsp
1429
1430         mov     120($context),%rax      # pull context->Rax
1431         mov     248($context),%rbx      # pull context->Rip
1432
1433         mov     8($disp),%rsi           # disp->ImageBase
1434         mov     56($disp),%r11          # disp->HandlerData
1435
1436         mov     0(%r11),%r10d           # HandlerData[0]
1437         lea     (%rsi,%r10),%r10        # end of prologue label
1438         cmp     %r10,%rbx               # context->Rip<body label
1439         jb      .Lin_prologue
1440
1441         mov     152($context),%rax      # pull context->Rsp
1442
1443         mov     4(%r11),%r10d           # HandlerData[1]
1444         lea     (%rsi,%r10),%r10        # epilogue label
1445         cmp     %r10,%rbx               # context->Rip>=epilogue label
1446         jae     .Lin_prologue
1447
1448         mov     `32*17`($context),%rax  # pull saved stack pointer
1449
1450         mov     -8(%rax),%rbx
1451         mov     -16(%rax),%rbp
1452         mov     -24(%rax),%r12
1453         mov     -32(%rax),%r13
1454         mov     -40(%rax),%r14
1455         mov     -48(%rax),%r15
1456         mov     %rbx,144($context)      # restore context->Rbx
1457         mov     %rbp,160($context)      # restore context->Rbp
1458         mov     %r12,216($context)      # restore cotnext->R12
1459         mov     %r13,224($context)      # restore cotnext->R13
1460         mov     %r14,232($context)      # restore cotnext->R14
1461         mov     %r15,240($context)      # restore cotnext->R15
1462
1463         lea     -56-10*16(%rax),%rsi
1464         lea     512($context),%rdi      # &context.Xmm6
1465         mov     \$20,%ecx
1466         .long   0xa548f3fc              # cld; rep movsq
1467
1468         jmp     .Lin_prologue
1469 .size   avx2_handler,.-avx2_handler
1470 ___
1471 $code.=<<___;
1472 .section        .pdata
1473 .align  4
1474         .rva    .LSEH_begin_sha1_multi_block
1475         .rva    .LSEH_end_sha1_multi_block
1476         .rva    .LSEH_info_sha1_multi_block
1477         .rva    .LSEH_begin_sha1_multi_block_shaext
1478         .rva    .LSEH_end_sha1_multi_block_shaext
1479         .rva    .LSEH_info_sha1_multi_block_shaext
1480 ___
1481 $code.=<<___ if ($avx);
1482         .rva    .LSEH_begin_sha1_multi_block_avx
1483         .rva    .LSEH_end_sha1_multi_block_avx
1484         .rva    .LSEH_info_sha1_multi_block_avx
1485 ___
1486 $code.=<<___ if ($avx>1);
1487         .rva    .LSEH_begin_sha1_multi_block_avx2
1488         .rva    .LSEH_end_sha1_multi_block_avx2
1489         .rva    .LSEH_info_sha1_multi_block_avx2
1490 ___
1491 $code.=<<___;
1492 .section        .xdata
1493 .align  8
1494 .LSEH_info_sha1_multi_block:
1495         .byte   9,0,0,0
1496         .rva    se_handler
1497         .rva    .Lbody,.Lepilogue                       # HandlerData[]
1498 .LSEH_info_sha1_multi_block_shaext:
1499         .byte   9,0,0,0
1500         .rva    se_handler
1501         .rva    .Lbody_shaext,.Lepilogue_shaext # HandlerData[]
1502 ___
1503 $code.=<<___ if ($avx);
1504 .LSEH_info_sha1_multi_block_avx:
1505         .byte   9,0,0,0
1506         .rva    se_handler
1507         .rva    .Lbody_avx,.Lepilogue_avx               # HandlerData[]
1508 ___
1509 $code.=<<___ if ($avx>1);
1510 .LSEH_info_sha1_multi_block_avx2:
1511         .byte   9,0,0,0
1512         .rva    avx2_handler
1513         .rva    .Lbody_avx2,.Lepilogue_avx2             # HandlerData[]
1514 ___
1515 }
1516 ####################################################################
1517
1518 sub rex {
1519   local *opcode=shift;
1520   my ($dst,$src)=@_;
1521   my $rex=0;
1522
1523     $rex|=0x04                  if ($dst>=8);
1524     $rex|=0x01                  if ($src>=8);
1525     unshift @opcode,$rex|0x40   if ($rex);
1526 }
1527
1528 sub sha1rnds4 {
1529     if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1530       my @opcode=(0x0f,0x3a,0xcc);
1531         rex(\@opcode,$3,$2);
1532         push @opcode,0xc0|($2&7)|(($3&7)<<3);           # ModR/M
1533         my $c=$1;
1534         push @opcode,$c=~/^0/?oct($c):$c;
1535         return ".byte\t".join(',',@opcode);
1536     } else {
1537         return "sha1rnds4\t".@_[0];
1538     }
1539 }
1540
1541 sub sha1op38 {
1542     my $instr = shift;
1543     my %opcodelet = (
1544                 "sha1nexte" => 0xc8,
1545                 "sha1msg1"  => 0xc9,
1546                 "sha1msg2"  => 0xca     );
1547
1548     if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1549       my @opcode=(0x0f,0x38);
1550         rex(\@opcode,$2,$1);
1551         push @opcode,$opcodelet{$instr};
1552         push @opcode,0xc0|($1&7)|(($2&7)<<3);           # ModR/M
1553         return ".byte\t".join(',',@opcode);
1554     } else {
1555         return $instr."\t".@_[0];
1556     }
1557 }
1558
1559 foreach (split("\n",$code)) {
1560         s/\`([^\`]*)\`/eval($1)/ge;
1561
1562         s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo                or
1563         s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo             or
1564
1565         s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go          or
1566         s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go         or
1567         s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go    or
1568         s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go        or
1569         s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go            or
1570         s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;
1571
1572         print $_,"\n";
1573 }
1574
1575 close STDOUT;