# OpenSSL context it's used with Intel engine, but can also be used as
# drop-in replacement for crypto/aes/asm/aes-586.pl [see below for
# details].
+#
+# Performance.
+#
+# To start with see corresponding paragraph in aesni-x86_64.pl...
+# Instead of filling table similar to one found there I've chosen to
+# summarize *comparison* results for raw ECB, CTR and CBC benchmarks.
+# The simplified table below represents 32-bit performance relative
+# to 64-bit one in every given point. Ratios vary for different
+# encryption modes, therefore interval values.
+#
+# 16-byte 64-byte 256-byte 1-KB 8-KB
+# 53-67% 67-84% 91-94% 95-98% 97-99.5%
+#
+# Lower ratios for smaller block sizes are perfectly understandable,
+# because function call overhead is higher in 32-bit mode. Largest
+# 8-KB block performance is virtually same: 32-bit code is less than
+# 1% slower for ECB, CBC and CCM, and ~3% slower otherwise.
$PREFIX="aesni"; # if $PREFIX is set to "AES", the script
# generates drop-in replacement for
$in1="xmm7"; $inout3="xmm7";
\f
# Inline version of internal aesni_[en|de]crypt1
+{ my $sn;
sub aesni_inline_generate1
-{ my $p=shift;
+{ my ($p,$inout)=@_; $inout=$inout0 if (!defined($inout));
+ $sn++;
&$movekey ($rndkey0,&QWP(0,$key));
&$movekey ($rndkey1,&QWP(16,$key));
&lea ($key,&DWP(32,$key));
- &pxor ($inout0,$rndkey0);
- &set_label("${p}1_loop");
- eval"&aes${p} ($inout0,$rndkey1)";
+ &pxor ($inout,$rndkey0);
+ &set_label("${p}1_loop_$sn");
+ eval"&aes${p} ($inout,$rndkey1)";
&dec ($rounds);
&$movekey ($rndkey1,&QWP(0,$key));
&lea ($key,&DWP(16,$key));
- &jnz (&label("${p}1_loop"));
- eval"&aes${p}last ($inout0,$rndkey1)";
-}
+ &jnz (&label("${p}1_loop_$sn"));
+ eval"&aes${p}last ($inout,$rndkey1)";
+}}
sub aesni_generate1 # fully unrolled loop
-{ my $p=shift;
+{ my ($p,$inout)=@_; $inout=$inout0 if (!defined($inout));
&function_begin_B("_aesni_${p}rypt1");
&$movekey ($rndkey0,&QWP(0,$key));
&$movekey ($rndkey1,&QWP(0x10,$key));
- &cmp ($rounds,11);
- &pxor ($inout0,$rndkey0);
+ &pxor ($inout,$rndkey0);
&$movekey ($rndkey0,&QWP(0x20,$key));
&lea ($key,&DWP(0x30,$key));
+ &cmp ($rounds,11);
&jb (&label("${p}128"));
&lea ($key,&DWP(0x20,$key));
&je (&label("${p}192"));
&lea ($key,&DWP(0x20,$key));
- eval"&aes${p} ($inout0,$rndkey1)";
+ eval"&aes${p} ($inout,$rndkey1)";
&$movekey ($rndkey1,&QWP(-0x40,$key));
- eval"&aes${p} ($inout0,$rndkey0)";
+ eval"&aes${p} ($inout,$rndkey0)";
&$movekey ($rndkey0,&QWP(-0x30,$key));
&set_label("${p}192");
- eval"&aes${p} ($inout0,$rndkey1)";
+ eval"&aes${p} ($inout,$rndkey1)";
&$movekey ($rndkey1,&QWP(-0x20,$key));
- eval"&aes${p} ($inout0,$rndkey0)";
+ eval"&aes${p} ($inout,$rndkey0)";
&$movekey ($rndkey0,&QWP(-0x10,$key));
&set_label("${p}128");
- eval"&aes${p} ($inout0,$rndkey1)";
+ eval"&aes${p} ($inout,$rndkey1)";
&$movekey ($rndkey1,&QWP(0,$key));
- eval"&aes${p} ($inout0,$rndkey0)";
+ eval"&aes${p} ($inout,$rndkey0)";
&$movekey ($rndkey0,&QWP(0x10,$key));
- eval"&aes${p} ($inout0,$rndkey1)";
+ eval"&aes${p} ($inout,$rndkey1)";
&$movekey ($rndkey1,&QWP(0x20,$key));
- eval"&aes${p} ($inout0,$rndkey0)";
+ eval"&aes${p} ($inout,$rndkey0)";
&$movekey ($rndkey0,&QWP(0x30,$key));
- eval"&aes${p} ($inout0,$rndkey1)";
+ eval"&aes${p} ($inout,$rndkey1)";
&$movekey ($rndkey1,&QWP(0x40,$key));
- eval"&aes${p} ($inout0,$rndkey0)";
+ eval"&aes${p} ($inout,$rndkey0)";
&$movekey ($rndkey0,&QWP(0x50,$key));
- eval"&aes${p} ($inout0,$rndkey1)";
+ eval"&aes${p} ($inout,$rndkey1)";
&$movekey ($rndkey1,&QWP(0x60,$key));
- eval"&aes${p} ($inout0,$rndkey0)";
+ eval"&aes${p} ($inout,$rndkey0)";
&$movekey ($rndkey0,&QWP(0x70,$key));
- eval"&aes${p} ($inout0,$rndkey1)";
- eval"&aes${p}last ($inout0,$rndkey0)";
+ eval"&aes${p} ($inout,$rndkey1)";
+ eval"&aes${p}last ($inout,$rndkey0)";
&ret();
&function_end_B("_aesni_${p}rypt1");
}
&pxor ($inout0,$rndkey0);
&pxor ($inout1,$rndkey0);
&pxor ($inout2,$rndkey0);
- &jmp (&label("${p}3_loop"));
- &set_label("${p}3_loop",16);
- eval"&aes${p} ($inout0,$rndkey1)";
&$movekey ($rndkey0,&QWP(0,$key));
+
+ &set_label("${p}3_loop");
+ eval"&aes${p} ($inout0,$rndkey1)";
eval"&aes${p} ($inout1,$rndkey1)";
&dec ($rounds);
eval"&aes${p} ($inout2,$rndkey1)";
&$movekey ($rndkey1,&QWP(16,$key));
eval"&aes${p} ($inout0,$rndkey0)";
- &lea ($key,&DWP(32,$key));
eval"&aes${p} ($inout1,$rndkey0)";
+ &lea ($key,&DWP(32,$key));
eval"&aes${p} ($inout2,$rndkey0)";
+ &$movekey ($rndkey0,&QWP(0,$key));
&jnz (&label("${p}3_loop"));
eval"&aes${p} ($inout0,$rndkey1)";
- &$movekey ($rndkey0,&QWP(0,$key));
eval"&aes${p} ($inout1,$rndkey1)";
eval"&aes${p} ($inout2,$rndkey1)";
eval"&aes${p}last ($inout0,$rndkey0)";
&pxor ($inout1,$rndkey0);
&pxor ($inout2,$rndkey0);
&pxor ($inout3,$rndkey0);
- &jmp (&label("${p}3_loop"));
- &set_label("${p}3_loop",16);
- eval"&aes${p} ($inout0,$rndkey1)";
&$movekey ($rndkey0,&QWP(0,$key));
+
+ &set_label("${p}3_loop");
+ eval"&aes${p} ($inout0,$rndkey1)";
eval"&aes${p} ($inout1,$rndkey1)";
&dec ($rounds);
eval"&aes${p} ($inout2,$rndkey1)";
eval"&aes${p} ($inout3,$rndkey1)";
&$movekey ($rndkey1,&QWP(16,$key));
eval"&aes${p} ($inout0,$rndkey0)";
- &lea ($key,&DWP(32,$key));
eval"&aes${p} ($inout1,$rndkey0)";
+ &lea ($key,&DWP(32,$key));
eval"&aes${p} ($inout2,$rndkey0)";
eval"&aes${p} ($inout3,$rndkey0)";
- &jnz (&label("${p}3_loop"));
+ &$movekey ($rndkey0,&QWP(0,$key));
+ &jnz (&label("${p}3_loop"));
+
eval"&aes${p} ($inout0,$rndkey1)";
- &$movekey ($rndkey0,&QWP(0,$key));
eval"&aes${p} ($inout1,$rndkey1)";
eval"&aes${p} ($inout2,$rndkey1)";
eval"&aes${p} ($inout3,$rndkey1)";
&movups ($inout1,&QWP(0x10,$inp));
&movups ($inout2,&QWP(0x20,$inp));
&call ("_aesni_encrypt3");
- &sub ($len,0x30);
&lea ($inp,&DWP(0x30,$inp));
- &lea ($out,&DWP(0x30,$out));
- &movups (&QWP(-0x30,$out),$inout0);
+ &movups (&QWP(0,$out),$inout0);
&mov ($key,$key_); # restore $key
- &movups (&QWP(-0x20,$out),$inout1);
+ &movups (&QWP(0x10,$out),$inout1);
&mov ($rounds,$rounds_); # restore $rounds
- &movups (&QWP(-0x10,$out),$inout2);
+ &movups (&QWP(0x20,$out),$inout2);
+ &lea ($out,&DWP(0x30,$out));
+ &sub ($len,0x30);
&ja (&label("ecb_enc_loop3"));
&add ($len,0x40);
- &jz (&label("ecb_ret"));
-
&set_label("ecb_enc_tail");
- &cmp ($len,0x20);
&movups ($inout0,&QWP(0,$inp));
+ &cmp ($len,0x20);
&jb (&label("ecb_enc_one"));
&movups ($inout1,&QWP(0x10,$inp));
&je (&label("ecb_enc_two"));
- &cmp ($len,0x30);
&movups ($inout2,&QWP(0x20,$inp));
+ &cmp ($len,0x30);
&je (&label("ecb_enc_three"));
&movups ($inout3,&QWP(0x30,$inp));
&call ("_aesni_encrypt4");
&jmp (&label("ecb_ret"));
&set_label("ecb_enc_two",16);
+ &pxor ($inout2,$inout2);
&call ("_aesni_encrypt3");
&movups (&QWP(0,$out),$inout0);
&movups (&QWP(0x10,$out),$inout1);
&movups ($inout1,&QWP(0x10,$inp));
&movups ($inout2,&QWP(0x20,$inp));
&call ("_aesni_decrypt3");
- &sub ($len,0x30);
&lea ($inp,&DWP(0x30,$inp));
- &lea ($out,&DWP(0x30,$out));
- &movups (&QWP(-0x30,$out),$inout0);
+ &movups (&QWP(0,$out),$inout0);
&mov ($key,$key_); # restore $key
- &movups (&QWP(-0x20,$out),$inout1);
+ &movups (&QWP(0x10,$out),$inout1);
&mov ($rounds,$rounds_); # restore $rounds
- &movups (&QWP(-0x10,$out),$inout2);
+ &movups (&QWP(0x20,$out),$inout2);
+ &lea ($out,&DWP(0x30,$out));
+ &sub ($len,0x30);
&ja (&label("ecb_dec_loop3"));
&add ($len,0x40);
- &jz (&label("ecb_ret"));
-
&set_label("ecb_dec_tail");
- &cmp ($len,0x20);
&movups ($inout0,&QWP(0,$inp));
+ &cmp ($len,0x20);
&jb (&label("ecb_dec_one"));
&movups ($inout1,&QWP(0x10,$inp));
&je (&label("ecb_dec_two"));
- &cmp ($len,0x30);
&movups ($inout2,&QWP(0x20,$inp));
+ &cmp ($len,0x30);
&je (&label("ecb_dec_three"));
&movups ($inout3,&QWP(0x30,$inp));
&call ("_aesni_decrypt4");
&jmp (&label("ecb_ret"));
&set_label("ecb_dec_two",16);
+ &pxor ($inout2,$inout2);
&call ("_aesni_decrypt3");
&movups (&QWP(0,$out),$inout0);
&movups (&QWP(0x10,$out),$inout1);
&function_end("aesni_ecb_encrypt");
\f
######################################################################
-# handles only complete blocks, operates on 32-bit counter and
-# does not update *ivec! (see engine/eng_aesni.c for details)
+# void aesni_ccm64_[en|de]crypt_blocks (const void *in, void *out,
+# size_t blocks, const AES_KEY *key,
+# const char *ivec,char *cmac);
+#
+# Handles only complete blocks, operates on 64-bit counter and
+# does not update *ivec! Nor does it finalize CMAC value
+# (see engine/eng_aesni.c for details)
#
+&function_begin("aesni_ccm64_encrypt_blocks");
+ &mov ($inp,&wparam(0));
+ &mov ($out,&wparam(1));
+ &mov ($len,&wparam(2));
+ &mov ($key,&wparam(3));
+ &mov ($rounds_,&wparam(4));
+ &mov ($rounds,&wparam(5));
+ &mov ($key_,"esp");
+ &sub ("esp",60);
+ &and ("esp",-16); # align stack
+ &mov (&DWP(48,"esp"),$key_);
+
+ &movdqu ($ivec,&QWP(0,$rounds_)); # load ivec
+ &movdqu ($inout1,&QWP(0,$rounds)); # load cmac
+
+ # compose byte-swap control mask for pshufb on stack
+ &mov (&DWP(0,"esp"),0x0c0d0e0f);
+ &mov (&DWP(4,"esp"),0x08090a0b);
+ &mov (&DWP(8,"esp"),0x04050607);
+ &mov (&DWP(12,"esp"),0x00010203);
+
+ # compose counter increment vector on stack
+ &mov ($rounds,1);
+ &xor ($key_,$key_);
+ &mov (&DWP(16,"esp"),$rounds);
+ &mov (&DWP(20,"esp"),$key_);
+ &mov (&DWP(24,"esp"),$key_);
+ &mov (&DWP(28,"esp"),$key_);
+
+ &movdqa ($inout3,&QWP(0,"esp"));
+ &pshufb ($ivec,$inout3); # keep iv in reverse order
+
+ &mov ($rounds,&DWP(240,$key));
+ &mov ($key_,$key);
+ &mov ($rounds_,$rounds);
+ &movdqa ($inout0,$ivec);
+
+&set_label("ccm64_enc_outer");
+ &movdqu ($in0,&QWP(0,$inp));
+ &pshufb ($inout0,$inout3);
+ &mov ($key,$key_);
+ &mov ($rounds,$rounds_);
+ &pxor ($inout1,$in0); # cmac^=inp
+ &pxor ($inout2,$inout2);
+
+ &call ("_aesni_encrypt3");
+
+ &paddq ($ivec,&QWP(16,"esp"));
+ &dec ($len);
+ &lea ($inp,&DWP(16,$inp));
+ &pxor ($in0,$inout0); # inp^=E(ivec)
+ &movdqa ($inout0,$ivec);
+ &movdqu (&QWP(0,$out),$in0);
+ &lea ($out,&DWP(16,$out));
+ &jnz (&label("ccm64_enc_outer"));
+
+ &mov ("esp",&DWP(48,"esp"));
+ &mov ($out,&wparam(5));
+ &movdqu (&QWP(0,$out),$inout1);
+&function_end("aesni_ccm64_encrypt_blocks");
+
+&function_begin("aesni_ccm64_decrypt_blocks");
+ &mov ($inp,&wparam(0));
+ &mov ($out,&wparam(1));
+ &mov ($len,&wparam(2));
+ &mov ($key,&wparam(3));
+ &mov ($rounds_,&wparam(4));
+ &mov ($rounds,&wparam(5));
+ &mov ($key_,"esp");
+ &sub ("esp",60);
+ &and ("esp",-16); # align stack
+ &mov (&DWP(48,"esp"),$key_);
+
+ &movdqu ($ivec,&QWP(0,$rounds_)); # load ivec
+ &movdqu ($inout1,&QWP(0,$rounds)); # load cmac
+
+ # compose byte-swap control mask for pshufb on stack
+ &mov (&DWP(0,"esp"),0x0c0d0e0f);
+ &mov (&DWP(4,"esp"),0x08090a0b);
+ &mov (&DWP(8,"esp"),0x04050607);
+ &mov (&DWP(12,"esp"),0x00010203);
+
+ # compose counter increment vector on stack
+ &mov ($rounds,1);
+ &xor ($key_,$key_);
+ &mov (&DWP(16,"esp"),$rounds);
+ &mov (&DWP(20,"esp"),$key_);
+ &mov (&DWP(24,"esp"),$key_);
+ &mov (&DWP(28,"esp"),$key_);
+
+ &movdqa ($inout3,&QWP(0,"esp")); # bswap mask
+ &movdqa ($inout0,$ivec);
+ &pshufb ($ivec,$inout3); # keep iv in reverse order
+
+ &mov ($rounds,&DWP(240,$key));
+ &mov ($key_,$key);
+ &mov ($rounds_,$rounds);
+
+ if ($inline)
+ { &aesni_inline_generate1("enc"); }
+ else
+ { &call ("_aesni_encrypt1"); }
+
+&set_label("ccm64_dec_outer");
+ &movdqu ($in0,&QWP(0,$inp));
+ &paddq ($ivec,&QWP(16,"esp"));
+ &dec ($len);
+ &lea ($inp,&QWP(16,$inp));
+ &pxor ($in0,$inout0);
+ &movdqa ($inout0,$ivec);
+ &mov ($key,$key_);
+ &mov ($rounds,$rounds_);
+ &pshufb ($inout0,$inout3);
+ &movdqu (&QWP(0,$out),$in0);
+ &lea ($out,&DWP(16,$out));
+
+ &jz (&label("ccm64_dec_break"));
+
+ &pxor ($inout2,$inout2);
+ &call ("_aesni_encrypt3");
+
+ &jmp (&label("ccm64_dec_outer"));
+
+&set_label("ccm64_dec_break",16);
+ if ($inline)
+ { &aesni_inline_generate1("enc",$inout1); }
+ else
+ { &call ("_aesni_encrypt1",$inout1); }
+
+ &mov ("esp",&DWP(48,"esp"));
+ &mov ($out,&wparam(5));
+ &movdqu (&QWP(0,$out),$inout1);
+&function_end("aesni_ccm64_decrypt_blocks");
+\f
+######################################################################
# void aesni_ctr32_encrypt_blocks (const void *in, void *out,
# size_t blocks, const AES_KEY *key,
# const char *ivec);
+#
+# Handles only complete blocks, operates on 32-bit counter and
+# does not update *ivec! (see engine/eng_aesni.c for details)
+#
&function_begin("aesni_ctr32_encrypt_blocks");
&mov ($inp,&wparam(0));
&mov ($out,&wparam(1));
&and ("esp",-16); # align stack
&mov (&DWP(48,"esp"),$key_);
+ &cmp ($len,1);
+ &je (&label("ctr32_one_shortcut"));
+
&movups ($inout3,&QWP(0,$rounds_)); # load ivec
# compose byte-swap control mask for pshufb on stack
&pinsrd ($inout3,$key_,3); # wipe 32-bit counter
&mov ($rounds,&DWP(240,$key)); # key->rounds
- &movaps ($rndkey0,&QWP(0,"esp")); # load byte-swap mask
+ &movdqa ($rndkey0,&QWP(0,"esp")); # load byte-swap mask
# $ivec is vector of 3 32-bit counters
&pxor ($ivec,$ivec);
&pinsrd ($ivec,$rounds_,1);
&inc ($rounds_);
&pinsrd ($ivec,$rounds_,2);
+ &pshufb ($ivec,$rndkey0); # byte swap
&cmp ($len,4);
- &pshufb ($ivec,$rndkey0); # byte swap
&jbe (&label("ctr32_tail"));
- &movaps (&QWP(32,"esp"),$inout3); # save counter-less ivec
+ &movdqa (&QWP(32,"esp"),$inout3); # save counter-less ivec
&mov ($rounds_,$rounds);
&mov ($key_,$key);
&sub ($len,4);
&set_label("ctr32_loop3",16);
&pshufd ($inout0,$ivec,3<<6); # place counter to upper dword
&pshufd ($inout1,$ivec,2<<6);
- &pshufd ($inout2,$ivec,1<<6);
&por ($inout0,$inout3); # merge counter-less ivec
+ &pshufd ($inout2,$ivec,1<<6);
&por ($inout1,$inout3);
&por ($inout2,$inout3);
- &call ("_aesni_encrypt3");
+ # inline _aesni_encrypt3 and interleave last round
+ # with own code...
+
+ &$movekey ($rndkey0,&QWP(0,$key));
+ &shr ($rounds,1);
+ &$movekey ($rndkey1,&QWP(16,$key));
+ &lea ($key,&DWP(32,$key));
+ &pxor ($inout0,$rndkey0);
+ &pxor ($inout1,$rndkey0);
+ &pxor ($inout2,$rndkey0);
+ &$movekey ($rndkey0,&QWP(0,$key));
+
+&set_label("ctr32_enc_loop3");
+ &aesenc ($inout0,$rndkey1);
+ &aesenc ($inout1,$rndkey1);
+ &dec ($rounds);
+ &aesenc ($inout2,$rndkey1);
+ &$movekey ($rndkey1,&QWP(16,$key));
+ &aesenc ($inout0,$rndkey0);
+ &aesenc ($inout1,$rndkey0);
+ &lea ($key,&DWP(32,$key));
+ &aesenc ($inout2,$rndkey0);
+ &$movekey ($rndkey0,&QWP(0,$key));
+ &jnz (&label("ctr32_enc_loop3"));
+
+ &aesenc ($inout0,$rndkey1);
+ &aesenc ($inout1,$rndkey1);
+ &aesenc ($inout2,$rndkey1);
+ &movdqa ($rndkey1,&QWP(0,"esp")); # load byte-swap mask
+
+ &aesenclast ($inout0,$rndkey0);
+ &pshufb ($ivec,$rndkey1); # byte swap
+ &movdqu ($in0,&QWP(0,$inp));
+ &aesenclast ($inout1,$rndkey0);
+ &paddd ($ivec,&QWP(16,"esp")); # counter increment
+ &movdqu ($in1,&QWP(0x10,$inp));
+ &aesenclast ($inout2,$rndkey0);
+ &pshufb ($ivec,$rndkey1); # byte swap
+ &movdqu ($rndkey0,&QWP(0x20,$inp));
+ &lea ($inp,&DWP(0x30,$inp));
- &movaps($rndkey0,&QWP(0,"esp")); # load byte-swap mask
- &movups ($in0,&QWP(0,$inp));
- &movups ($in1,&QWP(0x10,$inp));
- &movups ($rndkey1,&QWP(0x20,$inp));
- &pshufb($ivec,$rndkey0); # byte swap
- &paddd ($ivec,&QWP(16,"esp")); # counter increment
&pxor ($in0,$inout0);
+ &mov ($key,$key_);
&pxor ($in1,$inout1);
- &pxor ($rndkey1,$inout2);
- &movups (&QWP(0,$out),$in0);
- &movups (&QWP(0x10,$out),$in1);
- &movups (&QWP(0x20,$out),$rndkey1);
- &movaps ($inout3,&QWP(32,"esp")); # load counter-less ivec
- &pshufb($ivec,$rndkey0); # byte swap
+ &movdqu (&QWP(0,$out),$in0);
+ &pxor ($rndkey0,$inout2);
+ &movdqu (&QWP(0x10,$out),$in1);
+ &movdqu (&QWP(0x20,$out),$rndkey0);
+ &movdqa ($inout3,&QWP(32,"esp")); # load counter-less ivec
&sub ($len,3);
- &lea ($inp,&DWP(0x30,$inp));
&lea ($out,&DWP(0x30,$out));
- &mov ($key,$key_);
&mov ($rounds,$rounds_);
&ja (&label("ctr32_loop3"));
- &add ($len,4);
&pextrd ($rounds_,$ivec,1); # might need last counter value
- &jz (&label("ctr32_ret"));
+ &add ($len,4);
&bswap ($rounds_);
&set_label("ctr32_tail");
- &cmp ($len,2);
&pshufd ($inout0,$ivec,3<<6);
&pshufd ($inout1,$ivec,2<<6);
- &pshufd ($inout2,$ivec,1<<6);
&por ($inout0,$inout3);
+ &cmp ($len,2);
&jb (&label("ctr32_one"));
+ &lea ($rounds_,&DWP(1,$rounds_));
+ &pshufd ($inout2,$ivec,1<<6);
&por ($inout1,$inout3);
&je (&label("ctr32_two"));
- &cmp ($len,3);
+ &bswap ($rounds_);
&por ($inout2,$inout3);
+ &cmp ($len,3);
&je (&label("ctr32_three"));
- &inc ($rounds_); # compose last counter value
- &bswap ($rounds_);
- &pinsrd ($inout3,$rounds_,3);
+ &pinsrd ($inout3,$rounds_,3); # compose last counter value
&call ("_aesni_encrypt4");
- &movups ($in0,&QWP(0,$inp));
- &movups ($rndkey1,&QWP(0x10,$inp));
- &movups ($rndkey0,&QWP(0x20,$inp));
- &movups ($ivec,&QWP(0x30,$inp));
+ &movdqu ($in0,&QWP(0,$inp));
+ &movdqu ($rndkey1,&QWP(0x10,$inp));
&pxor ($in0,$inout0);
+ &movdqu ($rndkey0,&QWP(0x20,$inp));
&pxor ($rndkey1,$inout1);
+ &movdqu ($ivec,&QWP(0x30,$inp));
&pxor ($rndkey0,$inout2);
+ &movdqu (&QWP(0,$out),$in0);
&pxor ($ivec,$inout3);
- &movups (&QWP(0,$out),$in0);
- &movups (&QWP(0x10,$out),$rndkey1);
- &movups (&QWP(0x20,$out),$rndkey0);
- &movups (&QWP(0x30,$out),$ivec);
+ &movdqu (&QWP(0x10,$out),$rndkey1);
+ &movdqu (&QWP(0x20,$out),$rndkey0);
+ &movdqu (&QWP(0x30,$out),$ivec);
&jmp (&label("ctr32_ret"));
-&set_label("ctr32_one",16);
+&set_label("ctr32_one_shortcut",16);
+ &movdqu ($inout0,&QWP(0,$rounds_)); # load ivec
+ &mov ($rounds,&DWP(240,$key));
+
+&set_label("ctr32_one");
if ($inline)
{ &aesni_inline_generate1("enc"); }
else
{ &call ("_aesni_encrypt1"); }
- &movups ($in0,&QWP(0,$inp));
+ &movdqu ($in0,&QWP(0,$inp));
&pxor ($in0,$inout0);
- &movups (&QWP(0,$out),$in0);
+ &movdqu (&QWP(0,$out),$in0);
&jmp (&label("ctr32_ret"));
&set_label("ctr32_two",16);
+ &pxor ($inout2,$inout2);
&call ("_aesni_encrypt3");
- &movups ($in0,&QWP(0,$inp));
- &movups ($in1,&QWP(0x10,$inp));
+ &movdqu ($in0,&QWP(0,$inp));
+ &movdqu ($in1,&QWP(0x10,$inp));
&pxor ($in0,$inout0);
&pxor ($in1,$inout1);
- &movups (&QWP(0,$out),$in0);
- &movups (&QWP(0x10,$out),$in1);
+ &movdqu (&QWP(0,$out),$in0);
+ &movdqu (&QWP(0x10,$out),$in1);
&jmp (&label("ctr32_ret"));
&set_label("ctr32_three",16);
&call ("_aesni_encrypt3");
- &movups ($in0,&QWP(0,$inp));
- &movups ($in1,&QWP(0x10,$inp));
- &movups ($rndkey1,&QWP(0x20,$inp));
+ &movdqu ($in0,&QWP(0,$inp));
+ &movdqu ($in1,&QWP(0x10,$inp));
+ &movdqu ($rndkey1,&QWP(0x20,$inp));
&pxor ($in0,$inout0);
&pxor ($in1,$inout1);
+ &movdqu (&QWP(0,$out),$in0);
&pxor ($rndkey1,$inout2);
- &movups (&QWP(0,$out),$in0);
- &movups (&QWP(0x10,$out),$in1);
- &movups (&QWP(0x20,$out),$rndkey1);
+ &movdqu (&QWP(0x10,$out),$in1);
+ &movdqu (&QWP(0x20,$out),$rndkey1);
&set_label("ctr32_ret");
&mov ("esp",&DWP(48,"esp"));
&mov ($out,&wparam(1));
&mov ($len,&wparam(2));
&mov ($key,&wparam(3));
- &test ($len,$len);
&mov ($key_,&wparam(4));
+ &test ($len,$len);
&jz (&label("cbc_ret"));
&cmp (&wparam(5),0);
- &movups ($ivec,&QWP(0,$key_)); # load IV
+ &movdqu ($ivec,&QWP(0,$key_)); # load IV
&mov ($rounds,&DWP(240,$key));
&mov ($key_,$key); # backup $key
&mov ($rounds_,$rounds); # backup $rounds
&je (&label("cbc_decrypt"));
- &movaps ($inout0,$ivec);
+ &movdqa ($inout0,$ivec);
&cmp ($len,16);
&jb (&label("cbc_enc_tail"));
&sub ($len,16);
&jmp (&label("cbc_enc_loop"));
&set_label("cbc_enc_loop",16);
- &movups ($ivec,&QWP(0,$inp));
+ &movdqu ($ivec,&QWP(0,$inp));
&lea ($inp,&DWP(16,$inp));
&pxor ($inout0,$ivec);
if ($inline)
{ &aesni_inline_generate1("enc"); }
else
{ &call ("_aesni_encrypt1"); }
- &sub ($len,16);
- &lea ($out,&DWP(16,$out));
&mov ($rounds,$rounds_); # restore $rounds
&mov ($key,$key_); # restore $key
- &movups (&QWP(-16,$out),$inout0);
+ &movups (&QWP(0,$out),$inout0); # store output
+ &lea ($out,&DWP(16,$out));
+ &sub ($len,16);
&jnc (&label("cbc_enc_loop"));
&add ($len,16);
&jnz (&label("cbc_enc_tail"));
&movups ($inout2,&QWP(0x20,$inp));
&movaps ($in0,$inout0);
&movaps ($in1,$inout1);
+
&call ("_aesni_decrypt3");
- &sub ($len,0x30);
- &lea ($inp,&DWP(0x30,$inp));
- &lea ($out,&DWP(0x30,$out));
+
&pxor ($inout0,$ivec);
&pxor ($inout1,$in0);
- &movups ($ivec,&QWP(-0x10,$inp));
+ &movdqu ($ivec,&QWP(0x20,$inp));
+ &lea ($inp,&DWP(0x30,$inp));
&pxor ($inout2,$in1);
- &movups (&QWP(-0x30,$out),$inout0);
+ &movdqu (&QWP(0,$out),$inout0);
&mov ($rounds,$rounds_) # restore $rounds
- &movups (&QWP(-0x20,$out),$inout1);
+ &movdqu (&QWP(0x10,$out),$inout1);
&mov ($key,$key_); # restore $key
- &movups (&QWP(-0x10,$out),$inout2);
+ &movdqu (&QWP(0x20,$out),$inout2);
+ &lea ($out,&DWP(0x30,$out));
+ &sub ($len,0x30);
&ja (&label("cbc_dec_loop3"));
&add ($len,0x40);
- &jz (&label("cbc_ret"));
-
&set_label("cbc_dec_tail");
&movups ($inout0,&QWP(0,$inp));
- &cmp ($len,0x10);
&movaps ($in0,$inout0);
+ &cmp ($len,0x10);
&jbe (&label("cbc_dec_one"));
&movups ($inout1,&QWP(0x10,$inp));
- &cmp ($len,0x20);
&movaps ($in1,$inout1);
+ &cmp ($len,0x20);
&jbe (&label("cbc_dec_two"));
&movups ($inout2,&QWP(0x20,$inp));
&cmp ($len,0x30);
&jbe (&label("cbc_dec_three"));
&movups ($inout3,&QWP(0x30,$inp));
&call ("_aesni_decrypt4");
- &movups ($rndkey0,&QWP(0x10,$inp));
- &movups ($rndkey1,&QWP(0x20,$inp));
+ &movdqu ($rndkey0,&QWP(0x10,$inp));
+ &movdqu ($rndkey1,&QWP(0x20,$inp));
&pxor ($inout0,$ivec);
&pxor ($inout1,$in0);
- &movups ($ivec,&QWP(0x30,$inp));
- &movups (&QWP(0,$out),$inout0);
+ &movdqu ($ivec,&QWP(0x30,$inp));
+ &movdqu (&QWP(0,$out),$inout0);
&pxor ($inout2,$rndkey0);
&pxor ($inout3,$rndkey1);
- &movups (&QWP(0x10,$out),$inout1);
- &movups (&QWP(0x20,$out),$inout2);
- &movaps ($inout0,$inout3);
+ &movdqu (&QWP(0x10,$out),$inout1);
+ &movdqu (&QWP(0x20,$out),$inout2);
+ &movdqa ($inout0,$inout3);
&lea ($out,&DWP(0x30,$out));
&jmp (&label("cbc_dec_tail_collected"));
-&set_label("cbc_dec_one");
+&set_label("cbc_dec_one",16);
if ($inline)
{ &aesni_inline_generate1("dec"); }
else
{ &call ("_aesni_decrypt1"); }
&pxor ($inout0,$ivec);
- &movaps ($ivec,$in0);
+ &movdqa ($ivec,$in0);
&jmp (&label("cbc_dec_tail_collected"));
-&set_label("cbc_dec_two");
+&set_label("cbc_dec_two",16);
+ &pxor ($inout2,$inout2);
&call ("_aesni_decrypt3");
&pxor ($inout0,$ivec);
&pxor ($inout1,$in0);
- &movups (&QWP(0,$out),$inout0);
- &movaps ($inout0,$inout1);
- &movaps ($ivec,$in1);
+ &movdqu (&QWP(0,$out),$inout0);
+ &movdqa ($inout0,$inout1);
+ &movdqa ($ivec,$in1);
&lea ($out,&DWP(0x10,$out));
&jmp (&label("cbc_dec_tail_collected"));
-&set_label("cbc_dec_three");
+&set_label("cbc_dec_three",16);
&call ("_aesni_decrypt3");
&pxor ($inout0,$ivec);
&pxor ($inout1,$in0);
&pxor ($inout2,$in1);
- &movups (&QWP(0,$out),$inout0);
- &movups (&QWP(0x10,$out),$inout1);
- &movaps ($inout0,$inout2);
- &movups ($ivec,&QWP(0x20,$inp));
+ &movdqu (&QWP(0,$out),$inout0);
+ &movdqu (&QWP(0x10,$out),$inout1);
+ &movdqa ($inout0,$inout2);
+ &movdqu ($ivec,&QWP(0x20,$inp));
&lea ($out,&DWP(0x20,$out));
&set_label("cbc_dec_tail_collected");
&and ($len,15);
&jnz (&label("cbc_dec_tail_partial"));
- &movups (&QWP(0,$out),$inout0);
+ &movdqu (&QWP(0,$out),$inout0);
&jmp (&label("cbc_ret"));
-&set_label("cbc_dec_tail_partial");
+&set_label("cbc_dec_tail_partial",16);
&mov ($key_,"esp");
&sub ("esp",16);
&and ("esp",-16);
- &movaps (&QWP(0,"esp"),$inout0);
+ &movdqa (&QWP(0,"esp"),$inout0);
&mov ($inp,"esp");
&mov ("ecx",$len);
&data_word(0xA4F3F689); # rep movsb
&aesimc ("xmm1","xmm1");
&lea ($key,&DWP(16,$key));
&lea ("eax",&DWP(-16,"eax"));
- &cmp ("eax",$key);
&$movekey (&QWP(16,"eax"),"xmm0");
&$movekey (&QWP(-16,$key),"xmm1");
+ &cmp ("eax",$key);
&ja (&label("dec_key_inverse"));
&$movekey ("xmm0",&QWP(0,$key)); # inverse middle
# OpenSSL context it's used with Intel engine, but can also be used as
# drop-in replacement for crypto/aes/asm/aes-x86_64.pl [see below for
# details].
+#
+# Performance.
+#
+# Given aes(enc|dec) instructions' latency asymptotic performance for
+# non-parallelizable modes such as CBC encrypt is 3.75 cycles per byte
+# processed with 128-bit key. And given their throughput asymptotic
+# performance for parallelizable modes is 1.25 cycles per byte. Being
+# asymptotic limit it's not something you commonly achieve in reality,
+# but how close does one get? Below are results collected for
+# different modes and block sized. Pairs of numbers are for en-/
+# decryption.
+#
+# 16-byte 64-byte 256-byte 1-KB 8-KB
+# ECB 4.25/4.25 1.38/1.38 1.28/1.28 1.26/1.26 1.26/1.26
+# CTR 5.42/5.42 1.92/1.92 1.44/1.44 1.28/1.28 1.26/1.26
+# CBC 4.38/4.43 4.15/1.43 4.07/1.32 4.07/1.29 4.06/1.28
+# CCM 5.66/9.42 4.42/5.41 4.16/4.40 4.09/4.15 4.06/4.07
+# OFB 5.42/5.42 4.64/4.64 4.44/4.44 4.39/4.39 4.38/4.38
+# CFB 5.73/5.85 5.56/5.62 5.48/5.56 5.47/5.55 5.47/5.55
+#
+# ECB, CTR, CBC and CCM results are free from EVP overhead. This means
+# that otherwise used 'openssl speed -evp aes-128-??? -engine aesni
+# [-decrypt]' will exhibit 10-15% worse results for smaller blocks.
+# The results were collected with specially crafted speed.c benchmark
+# in order to compare them with results reported in "Intel Advanced
+# Encryption Standard (AES) New Instruction Set" White Paper Revision
+# 3.0 dated May 2010. All above results are consistently better. This
+# module also provides better performance for block sizes smaller than
+# 128 bytes in points *not* represented in the above table.
+#
+# Looking at the results for 8-KB buffer.
+#
+# CFB and OFB results are far from the limit, because implementation
+# uses "generic" CRYPTO_[c|o]fb128_encrypt interfaces relying on
+# single-block aesni_encrypt, which is not the most optimal way to go.
+# CBC encrypt result is unexpectedly high and there is no documented
+# explanation for it. Seemingly there is a small penalty for feeding
+# the result back to AES unit the way it's done in CBC mode. There is
+# nothing one can do and the result appears optimal. CCM result is
+# identical to CBC, because CBC-MAC is essentially CBC encrypt without
+# saving output. CCM CTR "stays invisible," because it's neatly
+# interleaved wih CBC-MAC. This provides ~30% improvement over
+# "straghtforward" CCM implementation with CTR and CBC-MAC performed
+# disjointly. Parallelizable modes practically achieve the theoretical
+# limit.
+#
+# Looking at how results vary with buffer size.
+#
+# Curves are practically saturated at 1-KB buffer size. In most cases
+# "256-byte" performance is >95%, and "64-byte" is ~90% of "8-KB" one.
+# CTR curve doesn't follow this pattern and is "slowest" changing one
+# with "256-byte" result being 87% of "8-KB." This is because overhead
+# in CTR mode is most computationally intensive. Small-block CCM
+# decrypt is slower than encrypt, because first CTR and last CBC-MAC
+# iterations can't be interleaved.
+#
+# Results for 192- and 256-bit keys.
+#
+# EVP-free results were observed to scale perfectly with number of
+# rounds for larger block sizes, i.e. 192-bit result being 10/12 times
+# lower and 256-bit one - 10/14. Well, in CBC encrypt case differences
+# are a tad smaller, because the above mentioned penalty biases all
+# results by same constant value. In similar way function call
+# overhead affects small-block performance, as well as OFB and CFB
+# results. Differences are not large, most common coefficients are
+# 10/11.7 and 10/13.4 (as opposite to 10/12.0 and 10/14.0), but one
+# observe even 10/11.2 and 10/12.4 (CTR, OFB, CFB, CTR)...
$PREFIX="aesni"; # if $PREFIX is set to "AES", the script
# generates drop-in replacement for
$out="%rsi";
$len="%rdx";
$key="%rcx"; # input to and changed by aesni_[en|de]cryptN !!!
-$ivp="%r8"; # cbc, ctr
+$ivp="%r8"; # cbc, ctr, ...
$rnds_="%r10d"; # backup copy for $rounds
$key_="%r11"; # backup copy for $key
$inout2="%xmm2"; $inout3="%xmm3";
$rndkey0="%xmm4"; $rndkey1="%xmm5";
-$iv="%xmm6"; $in0="%xmm7"; # used in CBC decrypt, CTR
+$iv="%xmm6"; $in0="%xmm7"; # used in CBC decrypt, CTR, ...
$in1="%xmm8"; $in2="%xmm9";
\f
# Inline version of internal aesni_[en|de]crypt1.
# cycles which take care of loop variables...
{ my $sn;
sub aesni_generate1 {
-my ($p,$key,$rounds)=@_;
+my ($p,$key,$rounds,$inout)=@_; $inout=$inout0 if (!defined($inout));
++$sn;
$code.=<<___;
$movkey ($key),$rndkey0
$movkey 16($key),$rndkey1
lea 32($key),$key
- pxor $rndkey0,$inout0
+ pxor $rndkey0,$inout
.Loop_${p}1_$sn:
- aes${p} $rndkey1,$inout0
+ aes${p} $rndkey1,$inout
dec $rounds
$movkey ($key),$rndkey1
lea 16($key),$key
jnz .Loop_${p}1_$sn # loop body is 16 bytes
- aes${p}last $rndkey1,$inout0
+ aes${p}last $rndkey1,$inout
___
}}
# void $PREFIX_[en|de]crypt (const void *inp,void *out,const AES_KEY *key);
pxor $rndkey0,$inout0
pxor $rndkey0,$inout1
pxor $rndkey0,$inout2
+ $movkey ($key),$rndkey0
.L${dir}_loop3:
aes${dir} $rndkey1,$inout0
- $movkey ($key),$rndkey0
aes${dir} $rndkey1,$inout1
dec $rounds
aes${dir} $rndkey1,$inout2
- aes${dir} $rndkey0,$inout0
$movkey 16($key),$rndkey1
+ aes${dir} $rndkey0,$inout0
aes${dir} $rndkey0,$inout1
lea 32($key),$key
aes${dir} $rndkey0,$inout2
+ $movkey ($key),$rndkey0
jnz .L${dir}_loop3
aes${dir} $rndkey1,$inout0
- $movkey ($key),$rndkey0
aes${dir} $rndkey1,$inout1
aes${dir} $rndkey1,$inout2
aes${dir}last $rndkey0,$inout0
pxor $rndkey0,$inout1
pxor $rndkey0,$inout2
pxor $rndkey0,$inout3
+ $movkey ($key),$rndkey0
.L${dir}_loop4:
aes${dir} $rndkey1,$inout0
- $movkey ($key),$rndkey0
aes${dir} $rndkey1,$inout1
dec $rounds
aes${dir} $rndkey1,$inout2
aes${dir} $rndkey1,$inout3
- aes${dir} $rndkey0,$inout0
$movkey 16($key),$rndkey1
+ aes${dir} $rndkey0,$inout0
aes${dir} $rndkey0,$inout1
lea 32($key),$key
aes${dir} $rndkey0,$inout2
aes${dir} $rndkey0,$inout3
+ $movkey ($key),$rndkey0
jnz .L${dir}_loop4
aes${dir} $rndkey1,$inout0
- $movkey ($key),$rndkey0
aes${dir} $rndkey1,$inout1
aes${dir} $rndkey1,$inout2
aes${dir} $rndkey1,$inout3
mov 240($key),$rounds # pull $rounds
and \$-16,$len
mov $key,$key_ # backup $key
- test %r8d,%r8d # 5th argument
mov $rounds,$rnds_ # backup $rounds
+ test %r8d,%r8d # 5th argument
jz .Lecb_decrypt
#--------------------------- ECB ENCRYPT ------------------------------#
cmp \$0x40,$len
movups 0x10($inp),$inout1
movups 0x20($inp),$inout2
call _aesni_encrypt3
- sub \$0x30,$len
lea 0x30($inp),$inp
- lea 0x30($out),$out
- movups $inout0,-0x30($out)
+ movups $inout0,($out)
mov $rnds_,$rounds # restore $rounds
- movups $inout1,-0x20($out)
+ movups $inout1,0x10($out)
mov $key_,$key # restore $key
- movups $inout2,-0x10($out)
+ movups $inout2,0x20($out)
+ lea 0x30($out),$out
+ sub \$0x30,$len
ja .Lecb_enc_loop3
add \$0x40,$len
- jz .Lecb_ret
-
.Lecb_enc_tail:
- cmp \$0x20,$len
movups ($inp),$inout0
+ cmp \$0x20,$len
jb .Lecb_enc_one
movups 0x10($inp),$inout1
je .Lecb_enc_two
- cmp \$0x30,$len
movups 0x20($inp),$inout2
+ cmp \$0x30,$len
je .Lecb_enc_three
movups 0x30($inp),$inout3
call _aesni_encrypt4
jmp .Lecb_ret
.align 16
.Lecb_enc_two:
+ pxor $inout2,$inout2
call _aesni_encrypt3
movups $inout0,($out)
movups $inout1,0x10($out)
movups 0x10($inp),$inout1
movups 0x20($inp),$inout2
call _aesni_decrypt3
- sub \$0x30,$len
lea 0x30($inp),$inp
- lea 0x30($out),$out
- movups $inout0,-0x30($out)
+ movups $inout0,($out)
mov $rnds_,$rounds # restore $rounds
- movups $inout1,-0x20($out)
+ movups $inout1,0x10($out)
mov $key_,$key # restore $key
- movups $inout2,-0x10($out)
+ movups $inout2,0x20($out)
+ lea 0x30($out),$out
+ sub \$0x30,$len
ja .Lecb_dec_loop3
add \$0x40,$len
- jz .Lecb_ret
-
.Lecb_dec_tail:
- cmp \$0x20,$len
movups ($inp),$inout0
+ cmp \$0x20,$len
jb .Lecb_dec_one
movups 0x10($inp),$inout1
je .Lecb_dec_two
- cmp \$0x30,$len
movups 0x20($inp),$inout2
+ cmp \$0x30,$len
je .Lecb_dec_three
movups 0x30($inp),$inout3
call _aesni_decrypt4
jmp .Lecb_ret
.align 16
.Lecb_dec_two:
+ pxor $inout2,$inout2
call _aesni_decrypt3
movups $inout0,($out)
movups $inout1,0x10($out)
ret
.size aesni_ecb_encrypt,.-aesni_ecb_encrypt
___
+\f
+{
######################################################################
-# handles only complete blocks, operates on 32-bit counter and
-# does not update *ivec! (see engine/eng_aesni.c for details)
+# void aesni_ccm64_[en|de]crypt_blocks (const void *in, void *out,
+# size_t blocks, const AES_KEY *key,
+# const char *ivec,char *cmac);
#
+# Handles only complete blocks, operates on 64-bit counter and
+# does not update *ivec! Nor does it finalize CMAC value
+# (see engine/eng_aesni.c for details)
+#
+{
+my $cmac="%r9"; # 6th argument
+
+my $increment="%xmm8";
+my $bswap_mask="%xmm9";
+
+$code.=<<___;
+.globl aesni_ccm64_encrypt_blocks
+.type aesni_ccm64_encrypt_blocks,\@function,6
+.align 16
+aesni_ccm64_encrypt_blocks:
+___
+$code.=<<___ if ($win64);
+ lea -0x58(%rsp),%rsp
+ movaps %xmm6,(%rsp)
+ movaps %xmm7,0x10(%rsp)
+ movaps %xmm8,0x20(%rsp)
+ movaps %xmm9,0x30(%rsp)
+.Lccm64_enc_body:
+___
+$code.=<<___;
+ movdqu ($ivp),$iv
+ movdqu ($cmac),$inout1
+ movdqa .Lincrement64(%rip),$increment
+ movdqa .Lbswap_mask(%rip),$bswap_mask
+ pshufb $bswap_mask,$iv # keep iv in reverse order
+
+ mov 240($key),$rounds # key->rounds
+ mov $key,$key_
+ mov $rounds,$rnds_
+ movdqa $iv,$inout0
+
+.Lccm64_enc_outer:
+ movdqu ($inp),$in0 # load inp
+ pshufb $bswap_mask,$inout0
+ mov $key_,$key
+ mov $rnds_,$rounds
+ pxor $in0,$inout1 # cmac^=inp
+ pxor $inout2,$inout2
+
+ call _aesni_encrypt3
+
+ paddq $increment,$iv
+ dec $len
+ lea 16($inp),$inp
+ pxor $inout0,$in0 # inp ^= E(iv)
+ movdqa $iv,$inout0
+ movdqu $in0,($out) # save output
+ lea 16($out),$out
+ jnz .Lccm64_enc_outer
+
+ movdqu $inout1,($cmac)
+___
+$code.=<<___ if ($win64);
+ movaps (%rsp),%xmm6
+ movaps 0x10(%rsp),%xmm7
+ movaps 0x20(%rsp),%xmm8
+ movaps 0x30(%rsp),%xmm9
+ lea 0x58(%rsp),%rsp
+.Lccm64_enc_ret:
+___
+$code.=<<___;
+ ret
+.size aesni_ccm64_encrypt_blocks,.-aesni_ccm64_encrypt_blocks
+___
+######################################################################
+$code.=<<___;
+.globl aesni_ccm64_decrypt_blocks
+.type aesni_ccm64_decrypt_blocks,\@function,6
+.align 16
+aesni_ccm64_decrypt_blocks:
+___
+$code.=<<___ if ($win64);
+ lea -0x58(%rsp),%rsp
+ movaps %xmm6,(%rsp)
+ movaps %xmm7,0x10(%rsp)
+ movaps %xmm8,0x20(%rsp)
+ movaps %xmm9,0x30(%rsp)
+.Lccm64_dec_body:
+___
+$code.=<<___;
+ movdqu ($ivp),$iv
+ movdqu ($cmac),$inout1
+ movdqa .Lincrement64(%rip),$increment
+ movdqa .Lbswap_mask(%rip),$bswap_mask
+
+ mov 240($key),$rounds # key->rounds
+ movdqa $iv,$inout0
+ pshufb $bswap_mask,$iv # keep iv in reverse order
+ mov $rounds,$rnds_
+ mov $key,$key_
+___
+ &aesni_generate1("enc",$key,$rounds);
+$code.=<<___;
+.Lccm64_dec_outer:
+ movdqu ($inp),$in0 # load inp
+ paddq $increment,$iv
+ dec $len
+ lea 16($inp),$inp
+ pxor $inout0,$in0
+ movdqa $iv,$inout0
+ mov $key_,$key
+ mov $rnds_,$rounds
+ pshufb $bswap_mask,$inout0
+ movdqu $in0,($out)
+ lea 16($out),$out
+ pxor $in0,$inout1 # cmac^=out
+
+ jz .Lccm64_dec_break
+
+ pxor $inout2,$inout2
+ call _aesni_encrypt3
+
+ jmp .Lccm64_dec_outer
+
+.align 16
+.Lccm64_dec_break:
+___
+ &aesni_generate1("enc",$key,$rounds,$inout1);
+$code.=<<___;
+ movdqu $inout1,($cmac)
+___
+$code.=<<___ if ($win64);
+ movaps (%rsp),%xmm6
+ movaps 0x10(%rsp),%xmm7
+ movaps 0x20(%rsp),%xmm8
+ movaps 0x30(%rsp),%xmm9
+ lea 0x58(%rsp),%rsp
+.Lccm64_dec_ret:
+___
+$code.=<<___;
+ ret
+.size aesni_ccm64_decrypt_blocks,.-aesni_ccm64_decrypt_blocks
+___
+}\f
+######################################################################
# void aesni_ctr32_encrypt_blocks (const void *in, void *out,
# size_t blocks, const AES_KEY *key,
# const char *ivec);
-$increment="%xmm10";
-$bswap_mask="%xmm11";
+#
+# Handles only complete blocks, operates on 32-bit counter and
+# does not update *ivec! (see engine/eng_aesni.c for details)
+#
+my $increment="%xmm10";
+my $bswap_mask="%xmm11";
$code.=<<___;
.globl aesni_ctr32_encrypt_blocks
.Lctr32_body:
___
$code.=<<___;
- movups ($ivp),$inout3
- movaps .Lincrement(%rip),$increment
- movaps .Lbswap_mask(%rip),$bswap_mask
+ cmp \$1,$len
+ je .Lctr32_one_shortcut
+
+ movdqu ($ivp),$inout3
+ movdqa .Lincrement32(%rip),$increment
+ movdqa .Lbswap_mask(%rip),$bswap_mask
xor $rounds,$rounds
pextrd \$3,$inout3,$rnds_ # pull 32-bit counter
pinsrd \$3,$rounds,$inout3 # wipe 32-bit counter
pinsrd \$1,$rnds_,$iv
inc $rnds_
pinsrd \$2,$rnds_,$iv
+ pshufb $bswap_mask,$iv
cmp \$4,$len
- pshufb $bswap_mask,$iv
jbe .Lctr32_tail
mov $rounds,$rnds_
mov $key,$key_
sub \$4,$len
- jmp .Lctr32_loop3
-.align 16
.Lctr32_loop3:
pshufd \$`3<<6`,$iv,$inout0 # place counter to upper dword
pshufd \$`2<<6`,$iv,$inout1
- pshufd \$`1<<6`,$iv,$inout2
- movups ($inp),$in0
- movups 0x10($inp),$in1
- movups 0x20($inp),$in2
por $inout3,$inout0 # merge counter-less ivec
+ pshufd \$`1<<6`,$iv,$inout2
por $inout3,$inout1
por $inout3,$inout2
- pshufb $bswap_mask,$iv
- call _aesni_encrypt3
+ # inline _aesni_encrypt3 and interleave last round
+ # with own code...
- paddd $increment,$iv
+ $movkey ($key),$rndkey0
+ shr \$1,$rounds
+ $movkey 16($key),$rndkey1
+ lea 32($key),$key
+ pxor $rndkey0,$inout0
+ pxor $rndkey0,$inout1
+ pxor $rndkey0,$inout2
+ $movkey ($key),$rndkey0
+ jmp .Lctr32_enc_loop3
+.align 16
+.Lctr32_enc_loop3:
+ aesenc $rndkey1,$inout0
+ aesenc $rndkey1,$inout1
+ dec $rounds
+ aesenc $rndkey1,$inout2
+ $movkey 16($key),$rndkey1
+ aesenc $rndkey0,$inout0
+ aesenc $rndkey0,$inout1
+ lea 32($key),$key
+ aesenc $rndkey0,$inout2
+ $movkey ($key),$rndkey0
+ jnz .Lctr32_enc_loop3
+
+ aesenc $rndkey1,$inout0
+ aesenc $rndkey1,$inout1
+ aesenc $rndkey1,$inout2
+
+ pshufb $bswap_mask,$iv
+ movdqu ($inp),$in0
+ aesenclast $rndkey0,$inout0
+ movdqu 0x10($inp),$in1
+ paddd $increment,$iv
+ aesenclast $rndkey0,$inout1
+ movdqu 0x20($inp),$in2
+ pshufb $bswap_mask,$iv
+ aesenclast $rndkey0,$inout2
+ lea 0x30($inp),$inp
+
+ mov $key_,$key
pxor $inout0,$in0
+ sub \$3,$len
+ mov $rnds_,$rounds
pxor $inout1,$in1
+ movdqu $in0,($out)
pxor $inout2,$in2
- pshufb $bswap_mask,$iv
- movups $in0,($out)
- movups $in1,0x10($out)
- movups $in2,0x20($out)
-
- sub \$3,$len
- lea 0x30($inp),$inp
+ movdqu $in1,0x10($out)
+ movdqu $in2,0x20($out)
lea 0x30($out),$out
- mov $key_,$key
- mov $rnds_,$rounds
ja .Lctr32_loop3
+ pextrd \$1,$iv,$rnds_ # might need last counter value
add \$4,$len
- pextrd \$1,$iv,$rnds_ # migh need last counter value
- jz .Lctr32_done
bswap $rnds_
.Lctr32_tail:
- cmp \$2,$len
pshufd \$`3<<6`,$iv,$inout0
pshufd \$`2<<6`,$iv,$inout1
- pshufd \$`1<<6`,$iv,$inout2
por $inout3,$inout0
- movups ($inp),$in0
+ movdqu ($inp),$in0
+ cmp \$2,$len
jb .Lctr32_one
+ lea 1($rnds_),$rnds_
+ pshufd \$`1<<6`,$iv,$inout2
por $inout3,$inout1
- movups 0x10($inp),$in1
+ movdqu 0x10($inp),$in1
je .Lctr32_two
- cmp \$3,$len
+ bswap $rnds_
por $inout3,$inout2
- movups 0x20($inp),$in2
+ movdqu 0x20($inp),$in2
+ cmp \$3,$len
je .Lctr32_three
- inc $rnds_ # compose last counter value
- bswap $rnds_
- pinsrd \$3,$rnds_,$inout3
- movups 0x30($inp),$iv
+ pinsrd \$3,$rnds_,$inout3 # compose last counter value
+ movdqu 0x30($inp),$iv
call _aesni_encrypt4
pxor $inout0,$in0
pxor $inout1,$in1
pxor $inout2,$in2
+ movdqu $in0,($out)
pxor $inout3,$iv
- movups $in0,($out)
- movups $in1,0x10($out)
- movups $in2,0x20($out)
- movups $iv,0x30($out)
+ movdqu $in1,0x10($out)
+ movdqu $in2,0x20($out)
+ movdqu $iv,0x30($out)
jmp .Lctr32_done
.align 16
+.Lctr32_one_shortcut:
+ movdqu ($ivp),$inout0
+ movdqu ($inp),$in0
+ mov 240($key),$rounds # key->rounds
.Lctr32_one:
___
&aesni_generate1("enc",$key,$rounds);
$code.=<<___;
pxor $inout0,$in0
- movups $in0,($out)
+ movdqu $in0,($out)
jmp .Lctr32_done
.align 16
.Lctr32_two:
+ pxor $inout2,$inout2
call _aesni_encrypt3
pxor $inout0,$in0
pxor $inout1,$in1
- movups $in0,($out)
- movups $in1,0x10($out)
+ movdqu $in0,($out)
+ movdqu $in1,0x10($out)
jmp .Lctr32_done
.align 16
pxor $inout0,$in0
pxor $inout1,$in1
pxor $inout2,$in2
- movups $in0,($out)
- movups $in1,0x10($out)
- movups $in2,0x20($out)
+ movdqu $in0,($out)
+ movdqu $in1,0x10($out)
+ movdqu $in2,0x20($out)
.Lctr32_done:
___
movaps 0x40(%rsp),%xmm10
movaps 0x50(%rsp),%xmm11
lea 0x68(%rsp),%rsp
+.Lctr32_ret:
___
$code.=<<___;
-.Lctr32_ret:
ret
.size aesni_ctr32_encrypt_blocks,.-aesni_ctr32_encrypt_blocks
___
-}
+}}
\f
########################################################################
# void $PREFIX_cbc_encrypt (const void *inp, void *out,
test %r9d,%r9d # 6th argument
jz .Lcbc_decrypt
#--------------------------- CBC ENCRYPT ------------------------------#
- movups ($ivp),$inout0 # load iv as initial state
- cmp \$16,$len
+ movdqu ($ivp),$inout0 # load iv as initial state
mov $rnds_,$rounds
+ cmp \$16,$len
jb .Lcbc_enc_tail
sub \$16,$len
jmp .Lcbc_enc_loop
-.align 16
+.align 16
.Lcbc_enc_loop:
- movups ($inp),$inout1 # load input
+ movdqu ($inp),$inout1 # load input
lea 16($inp),$inp
pxor $inout1,$inout0
___
&aesni_generate1("enc",$key,$rounds);
$code.=<<___;
- sub \$16,$len
- lea 16($out),$out
mov $rnds_,$rounds # restore $rounds
mov $key_,$key # restore $key
- movups $inout0,-16($out) # store output
+ movups $inout0,0($out) # store output
+ lea 16($out),$out
+ sub \$16,$len
jnc .Lcbc_enc_loop
add \$16,$len
jnz .Lcbc_enc_tail
___
$code.=<<___;
movups ($ivp),$iv
- cmp \$0x40,$len
mov $rnds_,$rounds
+ cmp \$0x40,$len
jbe .Lcbc_dec_tail
sub \$0x40,$len
jmp .Lcbc_dec_loop3
-.align 16
+.align 16
.Lcbc_dec_loop3:
movups ($inp),$inout0
movups 0x10($inp),$inout1
movaps $inout0,$in0
movaps $inout1,$in1
movaps $inout2,$in2
+
call _aesni_decrypt3
+
sub \$0x30,$len
lea 0x30($inp),$inp
lea 0x30($out),$out
pxor $in0,$inout1
movaps $in2,$iv
pxor $in1,$inout2
- movups $inout0,-0x30($out)
+ movdqu $inout0,-0x30($out)
mov $rnds_,$rounds # restore $rounds
- movups $inout1,-0x20($out)
+ movdqu $inout1,-0x20($out)
mov $key_,$key # restore $key
- movups $inout2,-0x10($out)
+ movdqu $inout2,-0x10($out)
ja .Lcbc_dec_loop3
add \$0x40,$len
movups $iv,($ivp)
- jz .Lcbc_dec_ret
-
.Lcbc_dec_tail:
movups ($inp),$inout0
- cmp \$0x10,$len
movaps $inout0,$in0
+ cmp \$0x10,$len
jbe .Lcbc_dec_one
movups 0x10($inp),$inout1
- cmp \$0x20,$len
movaps $inout1,$in1
+ cmp \$0x20,$len
jbe .Lcbc_dec_two
movups 0x20($inp),$inout2
- cmp \$0x30,$len
movaps $inout2,$in2
+ cmp \$0x30,$len
jbe .Lcbc_dec_three
movups 0x30($inp),$inout3
call _aesni_decrypt4
pxor $iv,$inout0
movups 0x30($inp),$iv
pxor $in0,$inout1
- movups $inout0,($out)
+ movdqu $inout0,($out)
pxor $in1,$inout2
- movups $inout1,0x10($out)
+ movdqu $inout1,0x10($out)
pxor $in2,$inout3
- movups $inout2,0x20($out)
- movaps $inout3,$inout0
+ movdqu $inout2,0x20($out)
+ movdqa $inout3,$inout0
lea 0x30($out),$out
jmp .Lcbc_dec_tail_collected
.align 16
jmp .Lcbc_dec_tail_collected
.align 16
.Lcbc_dec_two:
+ pxor $inout2,$inout2
call _aesni_decrypt3
pxor $iv,$inout0
pxor $in0,$inout1
- movups $inout0,($out)
+ movdqu $inout0,($out)
movaps $in1,$iv
- movaps $inout1,$inout0
+ movdqa $inout1,$inout0
lea 0x10($out),$out
jmp .Lcbc_dec_tail_collected
.align 16
call _aesni_decrypt3
pxor $iv,$inout0
pxor $in0,$inout1
- movups $inout0,($out)
+ movdqu $inout0,($out)
pxor $in1,$inout2
- movups $inout1,0x10($out)
+ movdqu $inout1,0x10($out)
movaps $in2,$iv
- movaps $inout2,$inout0
+ movdqa $inout2,$inout0
lea 0x20($out),$out
jmp .Lcbc_dec_tail_collected
.align 16
and \$15,$len
movups $iv,($ivp)
jnz .Lcbc_dec_tail_partial
- movups $inout0,($out)
+ movdqu $inout0,($out)
jmp .Lcbc_dec_ret
+.align 16
.Lcbc_dec_tail_partial:
movaps $inout0,$reserved(%rsp)
mov $out,%rdi
aesimc %xmm1,%xmm1
lea 16($key),$key
lea -16($inp),$inp
- cmp $key,$inp
$movkey %xmm0,16($inp)
$movkey %xmm1,-16($key)
+ cmp $key,$inp
ja .Ldec_key_inverse
$movkey ($key),%xmm0 # inverse middle
${PREFIX}_set_encrypt_key:
_aesni_set_encrypt_key:
.byte 0x48,0x83,0xEC,0x08 # sub rsp,8
- test $inp,$inp
mov \$-1,%rax
+ test $inp,$inp
jz .Lenc_key_ret
test $key,$key
jz .Lenc_key_ret
.align 64
.Lbswap_mask:
.byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
-.Lincrement:
+.Lincrement32:
.long 3,3,3,0
+.Lincrement64:
+ .long 1,0,0,0
.asciz "AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>"
.align 64
___
jmp .Lcommon_seh_exit
.size ecb_se_handler,.-ecb_se_handler
+.type ccm64_se_handler,\@abi-omnipotent
+.align 16
+ccm64_se_handler:
+ push %rsi
+ push %rdi
+ push %rbx
+ push %rbp
+ push %r12
+ push %r13
+ push %r14
+ push %r15
+ pushfq
+ sub \$64,%rsp
+
+ mov 120($context),%rax # pull context->Rax
+ mov 248($context),%rbx # pull context->Rip
+
+ mov 8($disp),%rsi # disp->ImageBase
+ mov 56($disp),$r11 # disp->HandlerData
+
+ mov 0(%r11),%r10d # HandlerData[0]
+ lea (%rsi,%r10),%r10 # prologue label
+ cmp %r10,%rbx # context->Rip<prologue label
+ jb .Lin_ccm64_prologue
+
+ mov 152($context),%rax # pull context->Rsp
+
+ mov 4(%r11),%r10d # HandlerData[1]
+ lea (%rsi,%r10),%r10 # epilogue label
+ cmp %r10,%rbx # context->Rip>=epilogue label
+ jae .Lin_ccm64_prologue
+
+ lea 0(%rax),%rsi # top of stack
+ lea 512($context),%rdi # &context.Xmm6
+ mov \$8,%ecx # 4*sizeof(%xmm0)/sizeof(%rax)
+ .long 0xa548f3fc # cld; rep movsq
+ lea 0x58(%rax),%rax # adjust stack pointer
+
+.Lin_ccm64_prologue:
+ mov 8(%rax),%rdi
+ mov 16(%rax),%rsi
+ mov %rax,152($context) # restore context->Rsp
+ mov %rsi,168($context) # restore context->Rsi
+ mov %rdi,176($context) # restore context->Rdi
+
+ jmp .Lcommon_seh_exit
+.size ccm64_se_handler,.-ccm64_se_handler
+
.type ctr32_se_handler,\@abi-omnipotent
.align 16
ctr32_se_handler:
.rva .LSEH_end_aesni_ecb_encrypt
.rva .LSEH_info_ecb
+ .rva .LSEH_begin_aesni_ccm64_encrypt_blocks
+ .rva .LSEH_end_aesni_ccm64_encrypt_blocks
+ .rva .LSEH_info_ccm64
+ .rva .Lccm64_enc_body,.Lccm64_enc_ret # HandlerData[]
+
+ .rva .LSEH_begin_aesni_ccm64_decrypt_blocks
+ .rva .LSEH_end_aesni_ccm64_decrypt_blocks
+ .rva .LSEH_info_ccm64
+ .rva .Lccm64_dec_body,.Lccm64_dec_ret # HandlerData[]
+
.rva .LSEH_begin_aesni_ctr32_encrypt_blocks
.rva .LSEH_end_aesni_ctr32_encrypt_blocks
.rva .LSEH_info_ctr32
.LSEH_info_ecb:
.byte 9,0,0,0
.rva ecb_se_handler
+.LSEH_info_ccm64:
+ .byte 9,0,0,0
+ .rva ccm64_se_handler
.LSEH_info_ctr32:
.byte 9,0,0,0
.rva ctr32_se_handler
.rva cbc_se_handler
.LSEH_info_key:
.byte 0x01,0x04,0x01,0x00
- .byte 0x04,0x02,0x00,0x00
+ .byte 0x04,0x02,0x00,0x00 # sub rsp,8
___
}