perlasm/x86_64-xlate.pl: work around problem with hex constants in masm.
[openssl.git] / crypto / perlasm / x86_64-xlate.pl
index 647e72e171d6e59b10fe99a4d757bac02bc5c8b2..645be9184d96853b5012aea5fc008327bc489abe 100755 (executable)
@@ -1,4 +1,11 @@
-#!/usr/bin/env perl
+#! /usr/bin/env perl
+# Copyright 2005-2016 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the OpenSSL license (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
 
 # Ascetic x86_64 AT&T to MASM/NASM assembler translator by <appro>.
 #
 # 6. Don't use [or hand-code with .byte] "rep ret." "ret" mnemonic is
 #    required to identify the spots, where to inject Win64 epilogue!
 #    But on the pros, it's then prefixed with rep automatically:-)
-# 7. Due to MASM limitations [and certain general counter-intuitivity
-#    of ip-relative addressing] generation of position-independent
-#    code is assisted by synthetic directive, .picmeup, which puts
-#    address of the *next* instruction into target register.
-#
-#    Example 1:
-#              .picmeup        %rax
-#              lea             .Label-.(%rax),%rax
-#    Example 2:
-#              .picmeup        %rcx
-#      .Lpic_point:
-#              ...
-#              lea             .Label-.Lpic_point(%rcx),%rbp
+# 7. Stick to explicit ip-relative addressing. If you have to use
+#    GOTPCREL addressing, stick to mov symbol@GOTPCREL(%rip),%r??.
+#    Both are recognized and translated to proper Win64 addressing
+#    modes.
 #
 # 8. In order to provide for structured exception handling unified
 #    Win64 prologue copies %rsp value to %rax. For further details
 #    see SEH paragraph at the end.
+# 9. .init segment is allowed to contain calls to functions only.
+# a. If function accepts more than 4 arguments *and* >4th argument
+#    is declared as non 64-bit value, do clear its upper part.
 \f
-my $output = shift;
 
-{ my ($stddev,$stdino,@junk)=stat(STDOUT);
-  my ($outdev,$outino,@junk)=stat($output);
+use strict;
 
-    open STDOUT,">$output" || die "can't open $output: $!"
-       if ($stddev!=$outdev || $stdino!=$outino);
-}
+my $flavour = shift;
+my $output  = shift;
+if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
 
-my $win64=1 if ($output =~ /\.asm$/);
+open STDOUT,">$output" || die "can't open $output: $!"
+       if (defined($output));
+
+my $gas=1;     $gas=0 if ($output =~ /\.asm$/);
+my $elf=1;     $elf=0 if (!$gas);
+my $win64=0;
+my $prefix="";
+my $decor=".L";
 
 my $masmref=8 + 50727*2**-32;  # 8.00.50727 shipped with VS2005
 my $masm=0;
@@ -77,12 +83,22 @@ my $PTR=" PTR";
 my $nasmref=2.03;
 my $nasm=0;
 
-if ($win64)
+if    ($flavour eq "mingw64")  { $gas=1; $elf=0; $win64=1;
+                                 $prefix=`echo __USER_LABEL_PREFIX__ | $ENV{CC} -E -P -`;
+                                 $prefix =~ s|\R$||; # Better chomp
+                               }
+elsif ($flavour eq "macosx")   { $gas=1; $elf=0; $prefix="_"; $decor="L\$"; }
+elsif ($flavour eq "masm")     { $gas=0; $elf=0; $masm=$masmref; $win64=1; $decor="\$L\$"; }
+elsif ($flavour eq "nasm")     { $gas=0; $elf=0; $nasm=$nasmref; $win64=1; $decor="\$L\$"; $PTR=""; }
+elsif (!$gas)
 {   if ($ENV{ASM} =~ m/nasm/ && `nasm -v` =~ m/version ([0-9]+)\.([0-9]+)/i)
     {  $nasm = $1 + $2*0.01; $PTR="";  }
     elsif (`ml64 2>&1` =~ m/Version ([0-9]+)\.([0-9]+)(\.([0-9]+))?/)
     {  $masm = $1 + $2*2**-16 + $4*2**-32;   }
-    die "no assembler found on %PATH" if (!($nasm || $masm));
+    die "no assembler found on %PATH%" if (!($nasm || $masm));
+    $win64=1;
+    $elf=0;
+    $decor="\$L\$";
 }
 
 my $current_segment;
@@ -91,21 +107,28 @@ my %globals;
 
 { package opcode;      # pick up opcodes
     sub re {
-       my      $self = shift;  # single instance in enough...
-       local   *line = shift;
-       undef   $ret;
+       my      ($class, $line) = @_;
+       my      $self = {};
+       my      $ret;
 
-       if ($line =~ /^([a-z][a-z0-9]*)/i) {
+       if ($$line =~ /^([a-z][a-z0-9]*)/i) {
+           bless $self,$class;
            $self->{op} = $1;
            $ret = $self;
-           $line = substr($line,@+[0]); $line =~ s/^\s+//;
+           $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;
 
            undef $self->{sz};
-           if ($self->{op} =~ /^(movz)b.*/) {  # movz is pain...
+           if ($self->{op} =~ /^(movz)x?([bw]).*/) {   # movz is pain...
                $self->{op} = $1;
-               $self->{sz} = "b";
+               $self->{sz} = $2;
            } elsif ($self->{op} =~ /call|jmp/) {
-               $self->{sz} = ""
+               $self->{sz} = "";
+           } elsif ($self->{op} =~ /^p/ && $' !~ /^(ush|op|insrw)/) { # SSEn
+               $self->{sz} = "";
+           } elsif ($self->{op} =~ /^[vk]/) { # VEX or k* such as kmov
+               $self->{sz} = "";
+           } elsif ($self->{op} =~ /mov[dq]/ && $$line =~ /%xmm/) {
+               $self->{sz} = "";
            } elsif ($self->{op} =~ /([a-z]{3,})([qlwb])$/) {
                $self->{op} = $1;
                $self->{sz} = $2;
@@ -114,20 +137,26 @@ my %globals;
        $ret;
     }
     sub size {
-       my $self = shift;
-       my $sz   = shift;
+       my ($self, $sz) = @_;
        $self->{sz} = $sz if (defined($sz) && !defined($self->{sz}));
        $self->{sz};
     }
     sub out {
        my $self = shift;
-       if (!$win64) {
+       if ($gas) {
            if ($self->{op} eq "movz") {        # movz is pain...
                sprintf "%s%s%s",$self->{op},$self->{sz},shift;
-           } elsif ($self->{op} =~ /^set/) { 
+           } elsif ($self->{op} =~ /^set/) {
                "$self->{op}";
            } elsif ($self->{op} eq "ret") {
-               ".byte  0xf3,0xc3";
+               my $epilogue = "";
+               if ($win64 && $current_function->{abi} eq "svr4") {
+                   $epilogue = "movq   8(%rsp),%rdi\n\t" .
+                               "movq   16(%rsp),%rsi\n\t";
+               }
+               $epilogue . ".byte      0xf3,0xc3";
+           } elsif ($self->{op} eq "call" && !$elf && $current_segment eq ".init") {
+               ".p2align\t3\n\t.quad";
            } else {
                "$self->{op}$self->{sz}";
            }
@@ -135,72 +164,100 @@ my %globals;
            $self->{op} =~ s/^movz/movzx/;
            if ($self->{op} eq "ret") {
                $self->{op} = "";
-               if ($current_function->{abi} eq "svr4") {
-                   $self->{op} = "mov  rdi,QWORD${PTR}[8+rsp]\t;WIN64 epilogue\n\t".
-                                 "mov  rsi,QWORD${PTR}[16+rsp]\n\t";
+               if ($win64 && $current_function->{abi} eq "svr4") {
+                   $self->{op} = "mov  rdi,QWORD$PTR\[8+rsp\]\t;WIN64 epilogue\n\t".
+                                 "mov  rsi,QWORD$PTR\[16+rsp\]\n\t";
                }
                $self->{op} .= "DB\t0F3h,0C3h\t\t;repret";
-           }
-           elsif ($self->{op} =~ /^(pop|push)f/) {
+           } elsif ($self->{op} =~ /^(pop|push)f/) {
                $self->{op} .= $self->{sz};
+           } elsif ($self->{op} eq "call" && $current_segment eq ".CRT\$XCU") {
+               $self->{op} = "\tDQ";
            }
            $self->{op};
        }
     }
-    sub mnemonic { shift->{op}; }
+    sub mnemonic {
+       my ($self, $op) = @_;
+       $self->{op}=$op if (defined($op));
+       $self->{op};
+    }
 }
 { package const;       # pick up constants, which start with $
     sub re {
-       my      $self = shift;  # single instance in enough...
-       local   *line = shift;
-       undef   $ret;
+       my      ($class, $line) = @_;
+       my      $self = {};
+       my      $ret;
 
-       if ($line =~ /^\$([^,]+)/) {
+       if ($$line =~ /^\$([^,]+)/) {
+           bless $self, $class;
            $self->{value} = $1;
            $ret = $self;
-           $line = substr($line,@+[0]); $line =~ s/^\s+//;
+           $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;
        }
        $ret;
     }
     sub out {
        my $self = shift;
 
-       if (!$win64) {
+       $self->{value} =~ s/\b(0b[0-1]+)/oct($1)/eig;
+       if ($gas) {
            # Solaris /usr/ccs/bin/as can't handle multiplications
            # in $self->{value}
-           $self->{value} =~ s/(?<![0-9a-f])(0[x0-9a-f]+)/oct($1)/egi;
-           $self->{value} =~ s/([0-9]+\s*[\*\/\%]\s*[0-9]+)/eval($1)/eg;
+           my $value = $self->{value};
+           no warnings;    # oct might complain about overflow, ignore here...
+           $value =~ s/(?<![\w\$\.])(0x?[0-9a-f]+)/oct($1)/egi;
+           if ($value =~ s/([0-9]+\s*[\*\/\%]\s*[0-9]+)/eval($1)/eg) {
+               $self->{value} = $value;
+           }
            sprintf "\$%s",$self->{value};
        } else {
-           $self->{value} =~ s/0x([0-9a-f]+)/0$1h/ig;
-           sprintf "%s",$self->{value};
+           my $value = $self->{value};
+           $value =~ s/0x([0-9a-f]+)/0$1h/ig if ($masm);
+           sprintf "%s",$value;
        }
     }
 }
 { package ea;          # pick up effective addresses: expr(%reg,%reg,scale)
+
+    my %szmap = (      b=>"BYTE$PTR",    w=>"WORD$PTR",
+                       l=>"DWORD$PTR",   d=>"DWORD$PTR",
+                       q=>"QWORD$PTR",   o=>"OWORD$PTR",
+                       x=>"XMMWORD$PTR", y=>"YMMWORD$PTR",
+                       z=>"ZMMWORD$PTR" ) if (!$gas);
+
     sub re {
-       my      $self = shift;  # single instance in enough...
-       local   *line = shift;
-       undef   $ret;
+       my      ($class, $line, $opcode) = @_;
+       my      $self = {};
+       my      $ret;
 
-       # optional * ---vvv--- appears in indirect jmp/call
-       if ($line =~ /^(\*?)([^\(,]*)\(([%\w,]+)\)/) {
+       # optional * ----vvv--- appears in indirect jmp/call
+       if ($$line =~ /^(\*?)([^\(,]*)\(([%\w,]+)\)((?:{[^}]+})*)/) {
+           bless $self, $class;
            $self->{asterisk} = $1;
            $self->{label} = $2;
            ($self->{base},$self->{index},$self->{scale})=split(/,/,$3);
            $self->{scale} = 1 if (!defined($self->{scale}));
+           $self->{opmask} = $4;
            $ret = $self;
-           $line = substr($line,@+[0]); $line =~ s/^\s+//;
+           $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;
 
+           if ($win64 && $self->{label} =~ s/\@GOTPCREL//) {
+               die if ($opcode->mnemonic() ne "mov");
+               $opcode->mnemonic("lea");
+           }
            $self->{base}  =~ s/^%//;
            $self->{index} =~ s/^%// if (defined($self->{index}));
+           $self->{opcode} = $opcode;
        }
        $ret;
     }
     sub size {}
     sub out {
-       my $self = shift;
-       my $sz = shift;
+       my ($self, $sz) = @_;
+
+       $self->{label} =~ s/([_a-z][_a-z0-9]*)/$globals{$1} or $1/gei;
+       $self->{label} =~ s/\.L/$decor/g;
 
        # Silently convert all EAs to 64-bit. This is required for
        # elder GNU assembler and results in more compact code,
@@ -208,63 +265,91 @@ my %globals;
        $self->{index} =~ s/^[er](.?[0-9xpi])[d]?$/r\1/;
        $self->{base}  =~ s/^[er](.?[0-9xpi])[d]?$/r\1/;
 
-       if (!$win64) {
-           # Solaris /usr/ccs/bin/as can't handle multiplications
-           # in $self->{label}
-           $self->{label} =~ s/(?<![0-9a-f])(0[x0-9a-f]+)/oct($1)/egi;
-           $self->{label} =~ s/([0-9]+\s*[\*\/\%]\s*[0-9]+)/eval($1)/eg;
+       # Solaris /usr/ccs/bin/as can't handle multiplications
+       # in $self->{label}...
+       use integer;
+       $self->{label} =~ s/(?<![\w\$\.])(0x?[0-9a-f]+)/oct($1)/egi;
+       $self->{label} =~ s/\b([0-9]+\s*[\*\/\%]\s*[0-9]+)\b/eval($1)/eg;
+
+       # Some assemblers insist on signed presentation of 32-bit
+       # offsets, but sign extension is a tricky business in perl...
+       if ((1<<31)<<1) {
+           $self->{label} =~ s/\b([0-9]+)\b/$1<<32>>32/eg;
+       } else {
+           $self->{label} =~ s/\b([0-9]+)\b/$1>>0/eg;
+       }
+
+       # if base register is %rbp or %r13, see if it's possible to
+       # flip base and index registers [for better performance]
+       if (!$self->{label} && $self->{index} && $self->{scale}==1 &&
+           $self->{base} =~ /(rbp|r13)/) {
+               $self->{base} = $self->{index}; $self->{index} = $1;
+       }
+
+       if ($gas) {
+           $self->{label} =~ s/^___imp_/__imp__/   if ($flavour eq "mingw64");
 
            if (defined($self->{index})) {
-               sprintf "%s%s(%%%s,%%%s,%d)",$self->{asterisk},
-                                       $self->{label},$self->{base},
-                                       $self->{index},$self->{scale};
+               sprintf "%s%s(%s,%%%s,%d)%s",
+                                       $self->{asterisk},$self->{label},
+                                       $self->{base}?"%$self->{base}":"",
+                                       $self->{index},$self->{scale},
+                                       $self->{opmask};
            } else {
-               sprintf "%s%s(%%%s)",   $self->{asterisk},$self->{label},$self->{base};
+               sprintf "%s%s(%%%s)%s", $self->{asterisk},$self->{label},
+                                       $self->{base},$self->{opmask};
            }
        } else {
-           %szmap = ( b=>"BYTE$PTR", w=>"WORD$PTR", l=>"DWORD$PTR", q=>"QWORD$PTR" );
-
-           $self->{label} =~ s/\.L/\$L\$/g;
            $self->{label} =~ s/\./\$/g;
-           $self->{label} =~ s/0x([0-9a-f]+)/0$1h/ig;
+           $self->{label} =~ s/(?<![\w\$\.])0x([0-9a-f]+)/0$1h/ig;
            $self->{label} = "($self->{label})" if ($self->{label} =~ /[\*\+\-\/]/);
-           $sz="q" if ($self->{asterisk});
+
+           my $mnemonic = $self->{opcode}->mnemonic();
+           ($self->{asterisk})                         && ($sz="q") ||
+           ($mnemonic =~ /^v?mov([qd])$/)              && ($sz=$1)  ||
+           ($mnemonic =~ /^v?pinsr([qdwb])$/)          && ($sz=$1)  ||
+           ($mnemonic =~ /^vpbroadcast([qdwb])$/)      && ($sz=$1)  ||
+           ($mnemonic =~ /^v(?!perm)[a-z]+[fi]128$/)   && ($sz="x");
+
+           $self->{opmask}  =~ s/%(k[0-7])/$1/;
 
            if (defined($self->{index})) {
-               sprintf "%s[%s%s*%d+%s]",$szmap{$sz},
+               sprintf "%s[%s%s*%d%s]%s",$szmap{$sz},
                                        $self->{label}?"$self->{label}+":"",
                                        $self->{index},$self->{scale},
-                                       $self->{base};
+                                       $self->{base}?"+$self->{base}":"",
+                                       $self->{opmask};
            } elsif ($self->{base} eq "rip") {
                sprintf "%s[%s]",$szmap{$sz},$self->{label};
            } else {
-               sprintf "%s[%s%s]",$szmap{$sz},
+               sprintf "%s[%s%s]%s",   $szmap{$sz},
                                        $self->{label}?"$self->{label}+":"",
-                                       $self->{base};
+                                       $self->{base},$self->{opmask};
            }
        }
     }
 }
 { package register;    # pick up registers, which start with %.
     sub re {
-       my      $class = shift; # muliple instances...
+       my      ($class, $line, $opcode) = @_;
        my      $self = {};
-       local   *line = shift;
-       undef   $ret;
+       my      $ret;
 
-       # optional * ---vvv--- appears in indirect jmp/call
-       if ($line =~ /^(\*?)%(\w+)/) {
+       # optional * ----vvv--- appears in indirect jmp/call
+       if ($$line =~ /^(\*?)%(\w+)((?:{[^}]+})*)/) {
            bless $self,$class;
            $self->{asterisk} = $1;
            $self->{value} = $2;
+           $self->{opmask} = $3;
+           $opcode->size($self->size());
            $ret = $self;
-           $line = substr($line,@+[0]); $line =~ s/^\s+//;
+           $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;
        }
        $ret;
     }
     sub size {
        my      $self = shift;
-       undef   $ret;
+       my      $ret;
 
        if    ($self->{value} =~ /^r[\d]+b$/i)  { $ret="b"; }
        elsif ($self->{value} =~ /^r[\d]+w$/i)  { $ret="w"; }
@@ -279,41 +364,63 @@ my %globals;
     }
     sub out {
        my $self = shift;
-       if (!$win64)    { sprintf "%s%%%s",$self->{asterisk},$self->{value}; }
-       else            { $self->{value}; }
+       if ($gas)       { sprintf "%s%%%s%s",   $self->{asterisk},
+                                               $self->{value},
+                                               $self->{opmask}; }
+       else            { $self->{opmask} =~ s/%(k[0-7])/$1/;
+                         $self->{value}.$self->{opmask}; }
     }
 }
 { package label;       # pick up labels, which end with :
     sub re {
-       my      $self = shift;  # single instance is enough...
-       local   *line = shift;
-       undef   $ret;
+       my      ($class, $line) = @_;
+       my      $self = {};
+       my      $ret;
 
-       if ($line =~ /(^[\.\w]+\:)/) {
+       if ($$line =~ /(^[\.\w]+)\:/) {
+           bless $self,$class;
            $self->{value} = $1;
            $ret = $self;
-           $line = substr($line,@+[0]); $line =~ s/^\s+//;
+           $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;
 
-           $self->{value} =~ s/\.L/\$L\$/ if ($win64);
+           $self->{value} =~ s/^\.L/$decor/;
        }
        $ret;
     }
     sub out {
        my $self = shift;
 
-       if (!$win64) {
-           $self->{value};
-       } elsif ($self->{value} ne "$current_function->{name}:") {
-           $self->{value} .= ":" if ($masm && $ret!~m/^\$/);
-           $self->{value};
-       } elsif ($current_function->{abi} eq "svr4") {
+       if ($gas) {
+           my $func = ($globals{$self->{value}} or $self->{value}) . ":";
+           if ($win64  && $current_function->{name} eq $self->{value}
+                       && $current_function->{abi} eq "svr4") {
+               $func .= "\n";
+               $func .= "      movq    %rdi,8(%rsp)\n";
+               $func .= "      movq    %rsi,16(%rsp)\n";
+               $func .= "      movq    %rsp,%rax\n";
+               $func .= "${decor}SEH_begin_$current_function->{name}:\n";
+               my $narg = $current_function->{narg};
+               $narg=6 if (!defined($narg));
+               $func .= "      movq    %rcx,%rdi\n" if ($narg>0);
+               $func .= "      movq    %rdx,%rsi\n" if ($narg>1);
+               $func .= "      movq    %r8,%rdx\n"  if ($narg>2);
+               $func .= "      movq    %r9,%rcx\n"  if ($narg>3);
+               $func .= "      movq    40(%rsp),%r8\n" if ($narg>4);
+               $func .= "      movq    48(%rsp),%r9\n" if ($narg>5);
+           }
+           $func;
+       } elsif ($self->{value} ne "$current_function->{name}") {
+           # Make all labels in masm global.
+           $self->{value} .= ":" if ($masm);
+           $self->{value} . ":";
+       } elsif ($win64 && $current_function->{abi} eq "svr4") {
            my $func =  "$current_function->{name}" .
                        ($nasm ? ":" : "\tPROC $current_function->{scope}") .
                        "\n";
-           $func .= "  mov     QWORD${PTR}[8+rsp],rdi\t;WIN64 prologue\n";
-           $func .= "  mov     QWORD${PTR}[16+rsp],rsi\n";
+           $func .= "  mov     QWORD$PTR\[8+rsp\],rdi\t;WIN64 prologue\n";
+           $func .= "  mov     QWORD$PTR\[16+rsp\],rsi\n";
            $func .= "  mov     rax,rsp\n";
-           $func .= "\$L\$SEH_begin_$current_function->{name}:";
+           $func .= "${decor}SEH_begin_$current_function->{name}:";
            $func .= ":" if ($masm);
            $func .= "\n";
            my $narg = $current_function->{narg};
@@ -322,8 +429,8 @@ my %globals;
            $func .= "  mov     rsi,rdx\n" if ($narg>1);
            $func .= "  mov     rdx,r8\n"  if ($narg>2);
            $func .= "  mov     rcx,r9\n"  if ($narg>3);
-           $func .= "  mov     r8,QWORD${PTR}[40+rsp]\n" if ($narg>4);
-           $func .= "  mov     r9,QWORD${PTR}[48+rsp]\n" if ($narg>5);
+           $func .= "  mov     r8,QWORD$PTR\[40+rsp\]\n" if ($narg>4);
+           $func .= "  mov     r9,QWORD$PTR\[48+rsp\]\n" if ($narg>5);
            $func .= "\n";
        } else {
           "$current_function->{name}".
@@ -331,68 +438,362 @@ my %globals;
        }
     }
 }
-{ package expr;                # pick up expressioins
+{ package expr;                # pick up expressions
     sub re {
-       my      $self = shift;  # single instance is enough...
-       local   *line = shift;
-       undef   $ret;
+       my      ($class, $line, $opcode) = @_;
+       my      $self = {};
+       my      $ret;
 
-       if ($line =~ /(^[^,]+)/) {
+       if ($$line =~ /(^[^,]+)/) {
+           bless $self,$class;
            $self->{value} = $1;
            $ret = $self;
-           $line = substr($line,@+[0]); $line =~ s/^\s+//;
+           $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;
 
-           $self->{value} =~ s/\.L/\$L\$/g if ($win64);
+           $self->{value} =~ s/\@PLT// if (!$elf);
+           $self->{value} =~ s/([_a-z][_a-z0-9]*)/$globals{$1} or $1/gei;
+           $self->{value} =~ s/\.L/$decor/g;
+           $self->{opcode} = $opcode;
        }
        $ret;
     }
     sub out {
        my $self = shift;
-       if ($nasm && opcode->mnemonic()=~m/^j/) {
+       if ($nasm && $self->{opcode}->mnemonic()=~m/^j(?![re]cxz)/) {
            "NEAR ".$self->{value};
        } else {
            $self->{value};
        }
     }
 }
+{ package cfi_directive;
+    # CFI directives annotate instructions that are significant for
+    # stack unwinding procedure compliant with DWARF specification,
+    # see http://dwarfstd.org/. Besides naturally expected for this
+    # script platform-specific filtering function, this module adds
+    # three auxiliary synthetic directives not recognized by [GNU]
+    # assembler:
+    #
+    # - .cfi_push to annotate push instructions in prologue, which
+    #   translates to .cfi_adjust_cfa_offset (if needed) and
+    #   .cfi_offset;
+    # - .cfi_pop to annotate pop instructions in epilogue, which
+    #   translates to .cfi_adjust_cfa_offset (if needed) and
+    #   .cfi_restore;
+    # - [and most notably] .cfi_cfa_expression which encodes
+    #   DW_CFA_def_cfa_expression and passes it to .cfi_escape as
+    #   byte vector;
+    #
+    # CFA expressions were introduced in DWARF specification version
+    # 3 and describe how to deduce CFA, Canonical Frame Address. This
+    # becomes handy if your stack frame is variable and you can't
+    # spare register for [previous] frame pointer. Suggested directive
+    # syntax is made-up mix of DWARF operator suffixes [subset of]
+    # and references to registers with optional bias. Following example
+    # describes offloaded *original* stack pointer at specific offset
+    # from *current* stack pointer:
+    #
+    #   .cfi_cfa_expression     %rsp+40,deref,+8
+    #
+    # Final +8 has everything to do with the fact that CFA is defined
+    # as reference to top of caller's stack, and on x86_64 call to
+    # subroutine pushes 8-byte return address. In other words original
+    # stack pointer upon entry to a subroutine is 8 bytes off from CFA.
+
+    # Below constants are taken from "DWARF Expressions" section of the
+    # DWARF specification, section is numbered 7.7 in versions 3 and 4.
+    my %DW_OP_simple = (       # no-arg operators, mapped directly
+       deref   => 0x06,        dup     => 0x12,
+       drop    => 0x13,        over    => 0x14,
+       pick    => 0x15,        swap    => 0x16,
+       rot     => 0x17,        xderef  => 0x18,
+
+       abs     => 0x19,        and     => 0x1a,
+       div     => 0x1b,        minus   => 0x1c,
+       mod     => 0x1d,        mul     => 0x1e,
+       neg     => 0x1f,        not     => 0x20,
+       or      => 0x21,        plus    => 0x22,
+       shl     => 0x24,        shr     => 0x25,
+       shra    => 0x26,        xor     => 0x27,
+       );
+
+    my %DW_OP_complex = (      # used in specific subroutines
+       constu          => 0x10,        # uleb128
+       consts          => 0x11,        # sleb128
+       plus_uconst     => 0x23,        # uleb128
+       lit0            => 0x30,        # add 0-31 to opcode
+       reg0            => 0x50,        # add 0-31 to opcode
+       breg0           => 0x70,        # add 0-31 to opcole, sleb128
+       regx            => 0x90,        # uleb28
+       fbreg           => 0x91,        # sleb128
+       bregx           => 0x92,        # uleb128, sleb128
+       piece           => 0x93,        # uleb128
+       );
+
+    # Following constants are defined in x86_64 ABI supplement, for
+    # example avaiable at https://www.uclibc.org/docs/psABI-x86_64.pdf,
+    # see section 3.7 "Stack Unwind Algorithm".
+    my %DW_reg_idx = (
+       "%rax"=>0,  "%rdx"=>1,  "%rcx"=>2,  "%rbx"=>3,
+       "%rsi"=>4,  "%rdi"=>5,  "%rbp"=>6,  "%rsp"=>7,
+       "%r8" =>8,  "%r9" =>9,  "%r10"=>10, "%r11"=>11,
+       "%r12"=>12, "%r13"=>13, "%r14"=>14, "%r15"=>15
+       );
+
+    my ($cfa_reg, $cfa_rsp);
+
+    # [us]leb128 format is variable-length integer representation base
+    # 2^128, with most significant bit of each byte being 0 denoting
+    # *last* most significat digit. See "Variable Length Data" in the
+    # DWARF specification, numbered 7.6 at least in versions 3 and 4.
+    sub sleb128 {
+       use integer;    # get right shift extend sign
+
+       my $val = shift;
+       my $sign = ($val < 0) ? -1 : 0;
+       my @ret = ();
+
+       while(1) {
+           push @ret, $val&0x7f;
+
+           # see if remaining bits are same and equal to most
+           # significant bit of the current digit, if so, it's
+           # last digit...
+           last if (($val>>6) == $sign);
+
+           @ret[-1] |= 0x80;
+           $val >>= 7;
+       }
+
+       return @ret;
+    }
+    sub uleb128 {
+       my $val = shift;
+       my @ret = ();
+
+       while(1) {
+           push @ret, $val&0x7f;
+
+           # see if it's last significant digit...
+           last if (($val >>= 7) == 0);
+
+           @ret[-1] |= 0x80;
+       }
+
+       return @ret;
+    }
+    sub const {
+       my $val = shift;
+
+       if ($val >= 0 && $val < 32) {
+            return ($DW_OP_complex{lit0}+$val);
+       }
+       return ($DW_OP_complex{consts}, sleb128($val));
+    }
+    sub reg {
+       my $val = shift;
+
+       return if ($val !~ m/^(%r\w+)(?:([\+\-])((?:0x)?[0-9a-f]+))?/);
+
+       my $reg = $DW_reg_idx{$1};
+       my $off = eval ("0 $2 $3");
+
+       return (($DW_OP_complex{breg0} + $reg), sleb128($off));
+       # Yes, we use DW_OP_bregX+0 to push register value and not
+       # DW_OP_regX, because latter would require even DW_OP_piece,
+       # which would be a waste under the circumstances. If you have
+       # to use DWP_OP_reg, use "regx:N"...
+    }
+    sub cfa_expression {
+       my $line = shift;
+       my @ret;
+
+       foreach my $token (split(/,\s*/,$line)) {
+           if ($token =~ /^%r/) {
+               push @ret,reg($token);
+           } elsif ($token =~ /((?:0x)?[0-9a-f]+)\((%r\w+)\)/) {
+               push @ret,reg("$2+$1");
+           } elsif ($token =~ /(\w+):(\-?(?:0x)?[0-9a-f]+)(U?)/i) {
+               my $i = 1*eval($2);
+               push @ret,$DW_OP_complex{$1}, ($3 ? uleb128($i) : sleb128($i));
+           } elsif (my $i = 1*eval($token) or $token eq "0") {
+               if ($token =~ /^\+/) {
+                   push @ret,$DW_OP_complex{plus_uconst},uleb128($i);
+               } else {
+                   push @ret,const($i);
+               }
+           } else {
+               push @ret,$DW_OP_simple{$token};
+           }
+       }
+
+       # Finally we return DW_CFA_def_cfa_expression, 15, followed by
+       # length of the expression and of course the expression itself.
+       return (15,scalar(@ret),@ret);
+    }
+    sub re {
+       my      ($class, $line) = @_;
+       my      $self = {};
+       my      $ret;
+
+       if ($$line =~ s/^\s*\.cfi_(\w+)\s*//) {
+           bless $self,$class;
+           $ret = $self;
+           undef $self->{value};
+           my $dir = $1;
+
+           SWITCH: for ($dir) {
+           # What is $cfa_rsp? Effectively it's difference between %rsp
+           # value and current CFA, Canonical Frame Address, which is
+           # why it starts with -8. Recall that CFA is top of caller's
+           # stack...
+           /startproc/ && do { ($cfa_reg, $cfa_rsp) = ("%rsp", -8); last; };
+           /endproc/   && do { ($cfa_reg, $cfa_rsp) = ("%rsp",  0); last; };
+           /def_cfa_register/
+                       && do { $cfa_reg = $$line; last; };
+           /def_cfa_offset/
+                       && do { $cfa_rsp = -1*eval($$line) if ($cfa_reg eq "%rsp");
+                               last;
+                             };
+           /adjust_cfa_offset/
+                       && do { $cfa_rsp -= 1*eval($$line) if ($cfa_reg eq "%rsp");
+                               last;
+                             };
+           /def_cfa/   && do { if ($$line =~ /(%r\w+)\s*,\s*(.+)/) {
+                                   $cfa_reg = $1;
+                                   $cfa_rsp = -1*eval($2) if ($cfa_reg eq "%rsp");
+                               }
+                               last;
+                             };
+           /push/      && do { $dir = undef;
+                               $cfa_rsp -= 8;
+                               if ($cfa_reg eq "%rsp") {
+                                   $self->{value} = ".cfi_adjust_cfa_offset\t8\n";
+                               }
+                               $self->{value} .= ".cfi_offset\t$$line,$cfa_rsp";
+                               last;
+                             };
+           /pop/       && do { $dir = undef;
+                               $cfa_rsp += 8;
+                               if ($cfa_reg eq "%rsp") {
+                                   $self->{value} = ".cfi_adjust_cfa_offset\t-8\n";
+                               }
+                               $self->{value} .= ".cfi_restore\t$$line";
+                               last;
+                             };
+           /cfa_expression/
+                       && do { $dir = undef;
+                               $self->{value} = ".cfi_escape\t" .
+                                       join(",", map(sprintf("0x%02x", $_),
+                                                     cfa_expression($$line)));
+                               last;
+                             };
+           }
+
+           $self->{value} = ".cfi_$dir\t$$line" if ($dir);
+
+           $$line = "";
+       }
+
+       return $ret;
+    }
+    sub out {
+       my $self = shift;
+       return ($elf ? $self->{value} : undef);
+    }
+}
 { package directive;   # pick up directives, which start with .
     sub re {
-       my      $self = shift;  # single instance is enough...
-       local   *line = shift;
-       undef   $ret;
+       my      ($class, $line) = @_;
+       my      $self = {};
+       my      $ret;
        my      $dir;
-       my      %opcode =       # lea 2f-1f(%rip),%dst; 1: nop; 2:
-               (       "%rax"=>0x01058d48,     "%rcx"=>0x010d8d48,
-                       "%rdx"=>0x01158d48,     "%rbx"=>0x011d8d48,
-                       "%rsp"=>0x01258d48,     "%rbp"=>0x012d8d48,
-                       "%rsi"=>0x01358d48,     "%rdi"=>0x013d8d48,
-                       "%r8" =>0x01058d4c,     "%r9" =>0x010d8d4c,
-                       "%r10"=>0x01158d4c,     "%r11"=>0x011d8d4c,
-                       "%r12"=>0x01258d4c,     "%r13"=>0x012d8d4c,
-                       "%r14"=>0x01358d4c,     "%r15"=>0x013d8d4c      );
-
-       if ($line =~ /^\s*(\.\w+)/) {
-           if (!$win64) {
-               $self->{value} = $1;
-               $line =~ s/\@abi\-omnipotent/\@function/;
-               $line =~ s/\@function.*/\@function/;
-               if ($line =~ /\.picmeup\s+(%r[\w]+)/i) {
-                   $self->{value} = sprintf "\t.long\t0x%x,0x90000000",$opcode{$1};
-               } elsif ($line =~ /\.asciz\s+"(.*)"$/) {
-                   $self->{value} = ".byte\t".join(",",unpack("C*",$1),0);
-               } elsif ($line =~ /\.extern/) {
+
+       # chain-call to cfi_directive
+       $ret = cfi_directive->re($line) and return $ret;
+
+       if ($$line =~ /^\s*(\.\w+)/) {
+           bless $self,$class;
+           $dir = $1;
+           $ret = $self;
+           undef $self->{value};
+           $$line = substr($$line,@+[0]); $$line =~ s/^\s+//;
+
+           SWITCH: for ($dir) {
+               /\.global|\.globl|\.extern/
+                           && do { $globals{$$line} = $prefix . $$line;
+                                   $$line = $globals{$$line} if ($prefix);
+                                   last;
+                                 };
+               /\.type/    && do { my ($sym,$type,$narg) = split(',',$$line);
+                                   if ($type eq "\@function") {
+                                       undef $current_function;
+                                       $current_function->{name} = $sym;
+                                       $current_function->{abi}  = "svr4";
+                                       $current_function->{narg} = $narg;
+                                       $current_function->{scope} = defined($globals{$sym})?"PUBLIC":"PRIVATE";
+                                   } elsif ($type eq "\@abi-omnipotent") {
+                                       undef $current_function;
+                                       $current_function->{name} = $sym;
+                                       $current_function->{scope} = defined($globals{$sym})?"PUBLIC":"PRIVATE";
+                                   }
+                                   $$line =~ s/\@abi\-omnipotent/\@function/;
+                                   $$line =~ s/\@function.*/\@function/;
+                                   last;
+                                 };
+               /\.asciz/   && do { if ($$line =~ /^"(.*)"$/) {
+                                       $dir  = ".byte";
+                                       $$line = join(",",unpack("C*",$1),0);
+                                   }
+                                   last;
+                                 };
+               /\.rva|\.long|\.quad/
+                           && do { $$line =~ s/([_a-z][_a-z0-9]*)/$globals{$1} or $1/gei;
+                                   $$line =~ s/\.L/$decor/g;
+                                   last;
+                                 };
+           }
+
+           if ($gas) {
+               $self->{value} = $dir . "\t" . $$line;
+
+               if ($dir =~ /\.extern/) {
                    $self->{value} = ""; # swallow extern
-               } else {
-                   $self->{value} = $line;
+               } elsif (!$elf && $dir =~ /\.type/) {
+                   $self->{value} = "";
+                   $self->{value} = ".def\t" . ($globals{$1} or $1) . ";\t" .
+                               (defined($globals{$1})?".scl 2;":".scl 3;") .
+                               "\t.type 32;\t.endef"
+                               if ($win64 && $$line =~ /([^,]+),\@function/);
+               } elsif (!$elf && $dir =~ /\.size/) {
+                   $self->{value} = "";
+                   if (defined($current_function)) {
+                       $self->{value} .= "${decor}SEH_end_$current_function->{name}:"
+                               if ($win64 && $current_function->{abi} eq "svr4");
+                       undef $current_function;
+                   }
+               } elsif (!$elf && $dir =~ /\.align/) {
+                   $self->{value} = ".p2align\t" . (log($$line)/log(2));
+               } elsif ($dir eq ".section") {
+                   $current_segment=$$line;
+                   if (!$elf && $current_segment eq ".init") {
+                       if      ($flavour eq "macosx")  { $self->{value} = ".mod_init_func"; }
+                       elsif   ($flavour eq "mingw64") { $self->{value} = ".section\t.ctors"; }
+                   }
+               } elsif ($dir =~ /\.(text|data)/) {
+                   $current_segment=".$1";
+               } elsif ($dir =~ /\.hidden/) {
+                   if    ($flavour eq "macosx")  { $self->{value} = ".private_extern\t$prefix$$line"; }
+                   elsif ($flavour eq "mingw64") { $self->{value} = ""; }
+               } elsif ($dir =~ /\.comm/) {
+                   $self->{value} = "$dir\t$prefix$$line";
+                   $self->{value} =~ s|,([0-9]+),([0-9]+)$|",$1,".log($2)/log(2)|e if ($flavour eq "macosx");
                }
-               $line = "";
+               $$line = "";
                return $self;
            }
 
-           $dir = $1;
-           $ret = $self;
-           undef $self->{value};
-           $line = substr($line,@+[0]); $line =~ s/^\s+//;
+           # non-gas case or nasm/masm
            SWITCH: for ($dir) {
                /\.text/    && do { my $v=undef;
                                    if ($nasm) {
@@ -401,7 +802,7 @@ my %globals;
                                        $v="$current_segment\tENDS\n" if ($current_segment);
                                        $current_segment = ".text\$";
                                        $v.="$current_segment\tSEGMENT ";
-                                       $v.=$masm>=$masmref ? "ALIGN(64)" : "PAGE";
+                                       $v.=$masm>=$masmref ? "ALIGN(256)" : "PAGE";
                                        $v.=" 'CODE'";
                                    }
                                    $self->{value} = $v;
@@ -419,96 +820,100 @@ my %globals;
                                    last;
                                  };
                /\.section/ && do { my $v=undef;
+                                   $$line =~ s/([^,]*).*/$1/;
+                                   $$line = ".CRT\$XCU" if ($$line eq ".init");
                                    if ($nasm) {
-                                       $v="section     $line";
-                                       if ($line=~/\.([px])data/) {
+                                       $v="section     $$line";
+                                       if ($$line=~/\.([px])data/) {
                                            $v.=" rdata align=";
                                            $v.=$1 eq "p"? 4 : 8;
+                                       } elsif ($$line=~/\.CRT\$/i) {
+                                           $v.=" rdata align=8";
                                        }
                                    } else {
                                        $v="$current_segment\tENDS\n" if ($current_segment);
-                                       $v.="$line\tSEGMENT";
-                                       if ($line=~/\.([px])data/) {
+                                       $v.="$$line\tSEGMENT";
+                                       if ($$line=~/\.([px])data/) {
                                            $v.=" READONLY";
                                            $v.=" ALIGN(".($1 eq "p" ? 4 : 8).")" if ($masm>=$masmref);
+                                       } elsif ($$line=~/\.CRT\$/i) {
+                                           $v.=" READONLY ";
+                                           $v.=$masm>=$masmref ? "ALIGN(8)" : "DWORD";
                                        }
                                    }
-                                   $current_segment = $line;
+                                   $current_segment = $$line;
                                    $self->{value} = $v;
                                    last;
                                  };
-               /\.extern/  && do { $self->{value}  = "EXTERN\t".$line;
+               /\.extern/  && do { $self->{value}  = "EXTERN\t".$$line;
                                    $self->{value} .= ":NEAR" if ($masm);
                                    last;
                                  };
                /\.globl|.global/
                            && do { $self->{value}  = $masm?"PUBLIC":"global";
-                                   $self->{value} .= "\t".$line;
-                                   $globals{$line} = $line;
-                                   last;
-                                 };
-               /\.type/    && do { ($sym,$type,$narg) = split(',',$line);
-                                   if ($type eq "\@function") {
-                                       undef $current_function;
-                                       $current_function->{name} = $sym;
-                                       $current_function->{abi}  = "svr4";
-                                       $current_function->{narg} = $narg;
-                                       $current_function->{scope} = defined($globals{$sym})?"PUBLIC":"PRIVATE";
-                                   } elsif ($type eq "\@abi-omnipotent") {
-                                       undef $current_function;
-                                       $current_function->{name} = $sym;
-                                       $current_function->{scope} = defined($globals{$sym})?"PUBLIC":"PRIVATE";
-                                   }
+                                   $self->{value} .= "\t".$$line;
                                    last;
                                  };
                /\.size/    && do { if (defined($current_function)) {
                                        undef $self->{value};
                                        if ($current_function->{abi} eq "svr4") {
-                                           $self->{value}="\$L\$SEH_end_$current_function->{name}:";
+                                           $self->{value}="${decor}SEH_end_$current_function->{name}:";
                                            $self->{value}.=":\n" if($masm);
                                        }
-                                       $self->{value}.="$current_function->{name}\tENDP" if($masm);
+                                       $self->{value}.="$current_function->{name}\tENDP" if($masm && $current_function->{name});
                                        undef $current_function;
                                    }
                                    last;
                                  };
-               /\.align/   && do { $self->{value} = "ALIGN\t".$line; last; };
-               /\.(byte|value|long|quad)/
+               /\.align/   && do { my $max = ($masm && $masm>=$masmref) ? 256 : 4096;
+                                   $self->{value} = "ALIGN\t".($$line>$max?$max:$$line);
+                                   last;
+                                 };
+               /\.(value|long|rva|quad)/
                            && do { my $sz  = substr($1,0,1);
-                                   my @arr = split(',',$line);
+                                   my @arr = split(/,\s*/,$$line);
                                    my $last = pop(@arr);
                                    my $conv = sub  {   my $var=shift;
-                                                       $var=~s/0x([0-9a-f]+)/0$1h/ig;
-                                                       $var=~s/\.L/\$L\$/g;
-                                                       if ($current_segment=~/.[px]data/)
+                                                       $var=~s/^(0b[0-1]+)/oct($1)/eig;
+                                                       $var=~s/^0x([0-9a-f]+)/0$1h/ig if ($masm);
+                                                       if ($sz eq "D" && ($current_segment=~/.[px]data/ || $dir eq ".rva"))
                                                        { $var=~s/([_a-z\$\@][_a-z0-9\$\@]*)/$nasm?"$1 wrt ..imagebase":"imagerel $1"/egi; }
                                                        $var;
-                                                   };  
+                                                   };
 
-                                   $sz =~ tr/bvlq/BWDQ/;
+                                   $sz =~ tr/bvlrq/BWDDQ/;
                                    $self->{value} = "\tD$sz\t";
                                    for (@arr) { $self->{value} .= &$conv($_).","; }
                                    $self->{value} .= &$conv($last);
                                    last;
                                  };
-               /\.picmeup/ && do { $self->{value} = sprintf"\tDD\t0%Xh,090000000h",$opcode{$line};
-                                   last;
-                                 };
-               /\.asciz/   && do { if ($line =~ /^"(.*)"$/) {
-                                       my @str=unpack("C*",$1);
-                                       push @str,0;
-                                       while ($#str>15) {
-                                           $self->{value}.="DB\t"
-                                               .join(",",@str[0..15])."\n";
-                                           foreach (0..15) { shift @str; }
-                                       }
+               /\.byte/    && do { my @str=split(/,\s*/,$$line);
+                                   map(s/(0b[0-1]+)/oct($1)/eig,@str);
+                                   map(s/0x([0-9a-f]+)/0$1h/ig,@str) if ($masm);
+                                   while ($#str>15) {
                                        $self->{value}.="DB\t"
+                                               .join(",",@str[0..15])."\n";
+                                       foreach (0..15) { shift @str; }
+                                   }
+                                   $self->{value}.="DB\t"
                                                .join(",",@str) if (@str);
+                                   last;
+                                 };
+               /\.comm/    && do { my @str=split(/,\s*/,$$line);
+                                   my $v=undef;
+                                   if ($nasm) {
+                                       $v.="common     $prefix@str[0] @str[1]";
+                                   } else {
+                                       $v="$current_segment\tENDS\n" if ($current_segment);
+                                       $current_segment = "_DATA";
+                                       $v.="$current_segment\tSEGMENT\n";
+                                       $v.="COMM       @str[0]:DWORD:".@str[1]/4;
                                    }
+                                   $self->{value} = $v;
                                    last;
                                  };
            }
-           $line = "";
+           $$line = "";
        }
 
        $ret;
@@ -519,63 +924,272 @@ my %globals;
     }
 }
 
+# Upon initial x86_64 introduction SSE>2 extensions were not introduced
+# yet. In order not to be bothered by tracing exact assembler versions,
+# but at the same time to provide a bare security minimum of AES-NI, we
+# hard-code some instructions. Extensions past AES-NI on the other hand
+# are traced by examining assembler version in individual perlasm
+# modules...
+
+my %regrm = (  "%eax"=>0, "%ecx"=>1, "%edx"=>2, "%ebx"=>3,
+               "%esp"=>4, "%ebp"=>5, "%esi"=>6, "%edi"=>7      );
+
+sub rex {
+ my $opcode=shift;
+ my ($dst,$src,$rex)=@_;
+
+   $rex|=0x04 if($dst>=8);
+   $rex|=0x01 if($src>=8);
+   push @$opcode,($rex|0x40) if ($rex);
+}
+
+my $movq = sub {       # elderly gas can't handle inter-register movq
+  my $arg = shift;
+  my @opcode=(0x66);
+    if ($arg =~ /%xmm([0-9]+),\s*%r(\w+)/) {
+       my ($src,$dst)=($1,$2);
+       if ($dst !~ /[0-9]+/)   { $dst = $regrm{"%e$dst"}; }
+       rex(\@opcode,$src,$dst,0x8);
+       push @opcode,0x0f,0x7e;
+       push @opcode,0xc0|(($src&7)<<3)|($dst&7);       # ModR/M
+       @opcode;
+    } elsif ($arg =~ /%r(\w+),\s*%xmm([0-9]+)/) {
+       my ($src,$dst)=($2,$1);
+       if ($dst !~ /[0-9]+/)   { $dst = $regrm{"%e$dst"}; }
+       rex(\@opcode,$src,$dst,0x8);
+       push @opcode,0x0f,0x6e;
+       push @opcode,0xc0|(($src&7)<<3)|($dst&7);       # ModR/M
+       @opcode;
+    } else {
+       ();
+    }
+};
+
+my $pextrd = sub {
+    if (shift =~ /\$([0-9]+),\s*%xmm([0-9]+),\s*(%\w+)/) {
+      my @opcode=(0x66);
+       my $imm=$1;
+       my $src=$2;
+       my $dst=$3;
+       if ($dst =~ /%r([0-9]+)d/)      { $dst = $1; }
+       elsif ($dst =~ /%e/)            { $dst = $regrm{$dst}; }
+       rex(\@opcode,$src,$dst);
+       push @opcode,0x0f,0x3a,0x16;
+       push @opcode,0xc0|(($src&7)<<3)|($dst&7);       # ModR/M
+       push @opcode,$imm;
+       @opcode;
+    } else {
+       ();
+    }
+};
+
+my $pinsrd = sub {
+    if (shift =~ /\$([0-9]+),\s*(%\w+),\s*%xmm([0-9]+)/) {
+      my @opcode=(0x66);
+       my $imm=$1;
+       my $src=$2;
+       my $dst=$3;
+       if ($src =~ /%r([0-9]+)/)       { $src = $1; }
+       elsif ($src =~ /%e/)            { $src = $regrm{$src}; }
+       rex(\@opcode,$dst,$src);
+       push @opcode,0x0f,0x3a,0x22;
+       push @opcode,0xc0|(($dst&7)<<3)|($src&7);       # ModR/M
+       push @opcode,$imm;
+       @opcode;
+    } else {
+       ();
+    }
+};
+
+my $pshufb = sub {
+    if (shift =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
+      my @opcode=(0x66);
+       rex(\@opcode,$2,$1);
+       push @opcode,0x0f,0x38,0x00;
+       push @opcode,0xc0|($1&7)|(($2&7)<<3);           # ModR/M
+       @opcode;
+    } else {
+       ();
+    }
+};
+
+my $palignr = sub {
+    if (shift =~ /\$([0-9]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
+      my @opcode=(0x66);
+       rex(\@opcode,$3,$2);
+       push @opcode,0x0f,0x3a,0x0f;
+       push @opcode,0xc0|($2&7)|(($3&7)<<3);           # ModR/M
+       push @opcode,$1;
+       @opcode;
+    } else {
+       ();
+    }
+};
+
+my $pclmulqdq = sub {
+    if (shift =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
+      my @opcode=(0x66);
+       rex(\@opcode,$3,$2);
+       push @opcode,0x0f,0x3a,0x44;
+       push @opcode,0xc0|($2&7)|(($3&7)<<3);           # ModR/M
+       my $c=$1;
+       push @opcode,$c=~/^0/?oct($c):$c;
+       @opcode;
+    } else {
+       ();
+    }
+};
+
+my $rdrand = sub {
+    if (shift =~ /%[er](\w+)/) {
+      my @opcode=();
+      my $dst=$1;
+       if ($dst !~ /[0-9]+/) { $dst = $regrm{"%e$dst"}; }
+       rex(\@opcode,0,$dst,8);
+       push @opcode,0x0f,0xc7,0xf0|($dst&7);
+       @opcode;
+    } else {
+       ();
+    }
+};
+
+my $rdseed = sub {
+    if (shift =~ /%[er](\w+)/) {
+      my @opcode=();
+      my $dst=$1;
+       if ($dst !~ /[0-9]+/) { $dst = $regrm{"%e$dst"}; }
+       rex(\@opcode,0,$dst,8);
+       push @opcode,0x0f,0xc7,0xf8|($dst&7);
+       @opcode;
+    } else {
+       ();
+    }
+};
+
+# Not all AVX-capable assemblers recognize AMD XOP extension. Since we
+# are using only two instructions hand-code them in order to be excused
+# from chasing assembler versions...
+
+sub rxb {
+ my $opcode=shift;
+ my ($dst,$src1,$src2,$rxb)=@_;
+
+   $rxb|=0x7<<5;
+   $rxb&=~(0x04<<5) if($dst>=8);
+   $rxb&=~(0x01<<5) if($src1>=8);
+   $rxb&=~(0x02<<5) if($src2>=8);
+   push @$opcode,$rxb;
+}
+
+my $vprotd = sub {
+    if (shift =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
+      my @opcode=(0x8f);
+       rxb(\@opcode,$3,$2,-1,0x08);
+       push @opcode,0x78,0xc2;
+       push @opcode,0xc0|($2&7)|(($3&7)<<3);           # ModR/M
+       my $c=$1;
+       push @opcode,$c=~/^0/?oct($c):$c;
+       @opcode;
+    } else {
+       ();
+    }
+};
+
+my $vprotq = sub {
+    if (shift =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
+      my @opcode=(0x8f);
+       rxb(\@opcode,$3,$2,-1,0x08);
+       push @opcode,0x78,0xc3;
+       push @opcode,0xc0|($2&7)|(($3&7)<<3);           # ModR/M
+       my $c=$1;
+       push @opcode,$c=~/^0/?oct($c):$c;
+       @opcode;
+    } else {
+       ();
+    }
+};
+
+# Intel Control-flow Enforcement Technology extension. All functions and
+# indirect branch targets will have to start with this instruction...
+
+my $endbranch = sub {
+    (0xf3,0x0f,0x1e,0xfa);
+};
+
+########################################################################
+
 if ($nasm) {
     print <<___;
 default        rel
+%define XMMWORD
+%define YMMWORD
+%define ZMMWORD
 ___
 } elsif ($masm) {
     print <<___;
 OPTION DOTNAME
 ___
 }
-while($line=<>) {
+while(defined(my $line=<>)) {
 
-    chomp($line);
+    $line =~ s|\R$||;           # Better chomp
 
     $line =~ s|[#!].*$||;      # get rid of asm-style comments...
     $line =~ s|/\*.*\*/||;     # ... and C-style comments...
     $line =~ s|^\s+||;         # ... and skip white spaces in beginning
+    $line =~ s|\s+$||;         # ... and at the end
 
-    undef $label;
-    undef $opcode;
-    undef $dst;
-    undef $src;
-    undef $sz;
+    if (my $label=label->re(\$line))   { print $label->out(); }
 
-    if ($label=label->re(\$line))      { print $label->out(); }
+    if (my $directive=directive->re(\$line)) {
+       printf "%s",$directive->out();
+    } elsif (my $opcode=opcode->re(\$line)) {
+       my $asm = eval("\$".$opcode->mnemonic());
 
-    if (directive->re(\$line)) {
-       printf "%s",directive->out();
-    } elsif ($opcode=opcode->re(\$line)) { ARGUMENT: {
+       if ((ref($asm) eq 'CODE') && scalar(my @bytes=&$asm($line))) {
+           print $gas?".byte\t":"DB\t",join(',',@bytes),"\n";
+           next;
+       }
 
-       if ($src=register->re(\$line))  { opcode->size($src->size()); }
-       elsif ($src=const->re(\$line))  { }
-       elsif ($src=ea->re(\$line))     { }
-       elsif ($src=expr->re(\$line))   { }
+       my @args;
+       ARGUMENT: while (1) {
+           my $arg;
 
-       last ARGUMENT if ($line !~ /^,/);
+           ($arg=register->re(\$line, $opcode))||
+           ($arg=const->re(\$line))            ||
+           ($arg=ea->re(\$line, $opcode))      ||
+           ($arg=expr->re(\$line, $opcode))    ||
+           last ARGUMENT;
 
-       $line = substr($line,1); $line =~ s/^\s+//;
+           push @args,$arg;
 
-       if ($dst=register->re(\$line))  { opcode->size($dst->size()); }
-       elsif ($dst=const->re(\$line))  { }
-       elsif ($dst=ea->re(\$line))     { }
+           last ARGUMENT if ($line !~ /^,/);
 
+           $line =~ s/^,\s*//;
        } # ARGUMENT:
 
-       $sz=opcode->size();
+       if ($#args>=0) {
+           my $insn;
+           my $sz=$opcode->size();
 
-       if (defined($dst)) {
-           if (!$win64) {
-               printf "\t%s\t%s,%s",   $opcode->out($dst->size()),
-                                       $src->out($sz),$dst->out($sz);
+           if ($gas) {
+               $insn = $opcode->out($#args>=1?$args[$#args]->size():$sz);
+               @args = map($_->out($sz),@args);
+               printf "\t%s\t%s",$insn,join(",",@args);
            } else {
+               $insn = $opcode->out();
+               foreach (@args) {
+                   my $arg = $_->out();
+                   # $insn.=$sz compensates for movq, pinsrw, ...
+                   if ($arg =~ /^xmm[0-9]+$/) { $insn.=$sz; $sz="x" if(!$sz); last; }
+                   if ($arg =~ /^ymm[0-9]+$/) { $insn.=$sz; $sz="y" if(!$sz); last; }
+                   if ($arg =~ /^zmm[0-9]+$/) { $insn.=$sz; $sz="z" if(!$sz); last; }
+                   if ($arg =~ /^mm[0-9]+$/)  { $insn.=$sz; $sz="q" if(!$sz); last; }
+               }
+               @args = reverse(@args);
                undef $sz if ($nasm && $opcode->mnemonic() eq "lea");
-               printf "\t%s\t%s,%s",   $opcode->out(),
-                                       $dst->out($sz),$src->out($sz);
+               printf "\t%s\t%s",$insn,join(",",map($_->out($sz),@args));
            }
-       } elsif (defined($src)) {
-           printf "\t%s\t%s",$opcode->out(),$src->out($sz);
        } else {
            printf "\t%s",$opcode->out();
        }
@@ -584,7 +1198,8 @@ while($line=<>) {
     print $line,"\n";
 }
 
-print "\n$current_segment\tENDS\nEND\n" if ($current_segment && $masm);
+print "\n$current_segment\tENDS\n"     if ($current_segment && $masm);
+print "END\n"                          if ($masm);
 
 close STDOUT;
 
@@ -608,13 +1223,13 @@ close STDOUT;
 # %r13         -               -
 # %r14         -               -
 # %r15         -               -
-# 
+#
 # (*)  volatile register
 # (-)  preserved by callee
 # (#)  Nth argument, volatile
 #
 # In Unix terms top of stack is argument transfer area for arguments
-# which could not be accomodated in registers. Or in other words 7th
+# which could not be accommodated in registers. Or in other words 7th
 # [integer] argument resides at 8(%rsp) upon function entry point.
 # 128 bytes above %rsp constitute a "red zone" which is not touched
 # by signal handlers and can be used as temporal storage without
@@ -631,7 +1246,7 @@ close STDOUT;
 # the area above user stack pointer in true asynchronous manner...
 #
 # All the above means that if assembler programmer adheres to Unix
-# register and stack layout, but disregards the "red zone" existense,
+# register and stack layout, but disregards the "red zone" existence,
 # it's possible to use following prologue and epilogue to "gear" from
 # Unix to Win64 ABI in leaf functions with not more than 6 arguments.
 #
@@ -689,6 +1304,7 @@ close STDOUT;
 #      movq    -16(%rcx),%rbx
 #      movq    -8(%rcx),%r15
 #      movq    %rcx,%rsp       # restore original rsp
+# magic_epilogue:
 #      ret
 # .size function,.-function
 #
@@ -701,11 +1317,16 @@ close STDOUT;
 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
 #              CONTEXT *context,DISPATCHER_CONTEXT *disp)
 # {    ULONG64 *rsp = (ULONG64 *)context->Rax;
-#      if (context->Rip >= magic_point)
-#      {   rsp = ((ULONG64 **)context->Rsp)[0];
-#          context->Rbp = rsp[-3];
-#          context->Rbx = rsp[-2];
-#          context->R15 = rsp[-1];
+#      ULONG64  rip = context->Rip;
+#
+#      if (rip >= magic_point)
+#      {   rsp = (ULONG64 *)context->Rsp;
+#          if (rip < magic_epilogue)
+#          {   rsp = (ULONG64 *)rsp[0];
+#              context->Rbp = rsp[-3];
+#              context->Rbx = rsp[-2];
+#              context->R15 = rsp[-1];
+#          }
 #      }
 #      context->Rsp = (ULONG64)rsp;
 #      context->Rdi = rsp[1];
@@ -740,6 +1361,7 @@ close STDOUT;
 #      CONTEXT.R14                             232
 #      CONTEXT.R15                             240
 #      CONTEXT.Rip                             248
+#      CONTEXT.Xmm6                            512
 #      sizeof(CONTEXT)                         1232
 #      DISPATCHER_CONTEXT.ControlPc            0
 #      DISPATCHER_CONTEXT.ImageBase            8
@@ -759,26 +1381,26 @@ close STDOUT;
 #
 # function_unwind_info:
 #      .byte   9,0,0,0
-#      .long   handler
+#      .rva    handler
 #
 # This structure designates exception handler for a function with
 # zero-length prologue, no stack frame or frame register.
 #
 # To facilitate composing of .pdata structures, auto-generated "gear"
 # prologue copies rsp value to rax and denotes next instruction with
-# $L$SEH_begin_{function_name} label. This essentially defines the SEH
+# .LSEH_begin_{function_name} label. This essentially defines the SEH
 # styling rule mentioned in the beginning. Position of this label is
 # chosen in such manner that possible exceptions raised in the "gear"
 # prologue would be accounted to caller and unwound from latter's frame.
-# End of function is marked with respective $L$SEH_end_{function_name}
+# End of function is marked with respective .LSEH_end_{function_name}
 # label. To summarize, .pdata segment would contain
 #
-#      .long   $L$SEH_begin_function
-#      .long   $L$SEH_end_function
-#      .long   function_unwind_info
+#      .rva    .LSEH_begin_function
+#      .rva    .LSEH_end_function
+#      .rva    function_unwind_info
 #
-# Reference to functon_unwind_info from .xdata segment is the anchor.
-# In case you wonder why references are 32-bit .longs and not 64-bit
+# Reference to function_unwind_info from .xdata segment is the anchor.
+# In case you wonder why references are 32-bit .rvas and not 64-bit
 # .quads. References put into these two segments are required to be
 # *relative* to the base address of the current binary module, a.k.a.
 # image base. No Win64 module, be it .exe or .dll, can be larger than
@@ -796,13 +1418,12 @@ close STDOUT;
 # instruction and reflecting it in finer grade unwind logic in handler.
 # After all, isn't it why it's called *language-specific* handler...
 #
-# Attentive reader can notice that exceptions would be mishandled in
-# auto-generated "gear" epilogue. Well, exception effectively can't
-# occur there, because if memory area used by it was subject to
-# segmentation violation, then it would be raised upon call to the
-# function (and as already mentioned be accounted to caller, which is
-# not a problem). If you're still not comfortable, then define tail
-# "magic point" just prior ret instruction and have handler treat it...
+# SE handlers are also involved in unwinding stack when executable is
+# profiled or debugged. Profiling implies additional limitations that
+# are too subtle to discuss here. For now it's sufficient to say that
+# in order to simplify handlers one should either a) offload original
+# %rsp to stack (like discussed above); or b) if you have a register to
+# spare for frame pointer, choose volatile one.
 #
 # (*)  Note that we're talking about run-time, not debug-time. Lack of
 #      unwind information makes debugging hard on both Windows and