# Add aesni_xts_[en|de]crypt. Westmere spends 1.50 cycles processing
# one byte out of 8KB with 128-bit key, Sandy Bridge - 1.09.
+# November 2015
+#
+# Add aesni_ocb_[en|de]crypt.
+
######################################################################
# Current large-block performance in cycles per byte processed with
# 128-bit key (less is better).
#
-# CBC en-/decrypt CTR XTS ECB
+# CBC en-/decrypt CTR XTS ECB OCB
# Westmere 3.77/1.37 1.37 1.52 1.27
-# * Bridge 5.07/0.98 0.99 1.09 0.91
-# Haswell 4.44/0.80 0.97 1.03 0.72
-# Atom 5.77/3.56 3.67 4.03 3.46
-# Bulldozer 5.80/0.98 1.05 1.24 0.93
+# * Bridge 5.07/0.98 0.99 1.09 0.91 1.10
+# Haswell 4.44/0.80 0.97 1.03 0.72 0.76
+# Silvermont 5.77/3.56 3.67 4.03 3.46 4.03
+# Bulldozer 5.80/0.98 1.05 1.24 0.93 1.23
$PREFIX="aesni"; # if $PREFIX is set to "AES", the script
# generates drop-in replacement for
&asm_init($ARGV[0],$0);
+&external_label("OPENSSL_ia32cap_P");
+&static_label("key_const");
+
if ($PREFIX eq "aesni") { $movekey=\&movups; }
else { $movekey=\&movups; }
$inout4="xmm6"; $in0="xmm6";
$inout5="xmm7"; $ivec="xmm7";
-# AESNI extenstion
+# AESNI extension
sub aeskeygenassist
{ my($dst,$src,$imm)=@_;
if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
{ &aesni_inline_generate1("enc"); }
else
{ &call ("_aesni_encrypt1"); }
+ &pxor ($rndkey0,$rndkey0); # clear register bank
+ &pxor ($rndkey1,$rndkey1);
&movups (&QWP(0,"eax"),$inout0);
+ &pxor ($inout0,$inout0);
&ret ();
&function_end_B("${PREFIX}_encrypt");
{ &aesni_inline_generate1("dec"); }
else
{ &call ("_aesni_decrypt1"); }
+ &pxor ($rndkey0,$rndkey0); # clear register bank
+ &pxor ($rndkey1,$rndkey1);
&movups (&QWP(0,"eax"),$inout0);
+ &pxor ($inout0,$inout0);
&ret ();
&function_end_B("${PREFIX}_decrypt");
&neg ($rounds);
eval"&aes${p} ($inout2,$rndkey1)";
&pxor ($inout5,$rndkey0);
+ &$movekey ($rndkey0,&QWP(0,$key,$rounds));
&add ($rounds,16);
- eval"&aes${p} ($inout3,$rndkey1)";
- eval"&aes${p} ($inout4,$rndkey1)";
- eval"&aes${p} ($inout5,$rndkey1)";
- &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
- &jmp (&label("_aesni_${p}rypt6_enter"));
+ &jmp (&label("_aesni_${p}rypt6_inner"));
&set_label("${p}6_loop",16);
eval"&aes${p} ($inout0,$rndkey1)";
eval"&aes${p} ($inout1,$rndkey1)";
eval"&aes${p} ($inout2,$rndkey1)";
+ &set_label("_aesni_${p}rypt6_inner");
eval"&aes${p} ($inout3,$rndkey1)";
eval"&aes${p} ($inout4,$rndkey1)";
eval"&aes${p} ($inout5,$rndkey1)";
&movups (&QWP(0x30,$out),$inout3);
&set_label("ecb_ret");
+ &pxor ("xmm0","xmm0"); # clear register bank
+ &pxor ("xmm1","xmm1");
+ &pxor ("xmm2","xmm2");
+ &pxor ("xmm3","xmm3");
+ &pxor ("xmm4","xmm4");
+ &pxor ("xmm5","xmm5");
+ &pxor ("xmm6","xmm6");
+ &pxor ("xmm7","xmm7");
&function_end("aesni_ecb_encrypt");
\f
######################################################################
&mov ("esp",&DWP(48,"esp"));
&mov ($out,&wparam(5));
&movups (&QWP(0,$out),$cmac);
+
+ &pxor ("xmm0","xmm0"); # clear register bank
+ &pxor ("xmm1","xmm1");
+ &pxor ("xmm2","xmm2");
+ &pxor ("xmm3","xmm3");
+ &pxor ("xmm4","xmm4");
+ &pxor ("xmm5","xmm5");
+ &pxor ("xmm6","xmm6");
+ &pxor ("xmm7","xmm7");
&function_end("aesni_ccm64_encrypt_blocks");
&function_begin("aesni_ccm64_decrypt_blocks");
&mov ("esp",&DWP(48,"esp"));
&mov ($out,&wparam(5));
&movups (&QWP(0,$out),$cmac);
+
+ &pxor ("xmm0","xmm0"); # clear register bank
+ &pxor ("xmm1","xmm1");
+ &pxor ("xmm2","xmm2");
+ &pxor ("xmm3","xmm3");
+ &pxor ("xmm4","xmm4");
+ &pxor ("xmm5","xmm5");
+ &pxor ("xmm6","xmm6");
+ &pxor ("xmm7","xmm7");
&function_end("aesni_ccm64_decrypt_blocks");
}
\f
&movups (&QWP(0x30,$out),$inout3);
&set_label("ctr32_ret");
+ &pxor ("xmm0","xmm0"); # clear register bank
+ &pxor ("xmm1","xmm1");
+ &pxor ("xmm2","xmm2");
+ &pxor ("xmm3","xmm3");
+ &pxor ("xmm4","xmm4");
+ &movdqa (&QWP(32,"esp"),"xmm0"); # clear stack
+ &pxor ("xmm5","xmm5");
+ &movdqa (&QWP(48,"esp"),"xmm0");
+ &pxor ("xmm6","xmm6");
+ &movdqa (&QWP(64,"esp"),"xmm0");
+ &pxor ("xmm7","xmm7");
&mov ("esp",&DWP(80,"esp"));
&function_end("aesni_ctr32_encrypt_blocks");
\f
&movups (&QWP(-16,$out),$inout0); # write output
&set_label("xts_enc_ret");
+ &pxor ("xmm0","xmm0"); # clear register bank
+ &pxor ("xmm1","xmm1");
+ &pxor ("xmm2","xmm2");
+ &movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack
+ &pxor ("xmm3","xmm3");
+ &movdqa (&QWP(16*1,"esp"),"xmm0");
+ &pxor ("xmm4","xmm4");
+ &movdqa (&QWP(16*2,"esp"),"xmm0");
+ &pxor ("xmm5","xmm5");
+ &movdqa (&QWP(16*3,"esp"),"xmm0");
+ &pxor ("xmm6","xmm6");
+ &movdqa (&QWP(16*4,"esp"),"xmm0");
+ &pxor ("xmm7","xmm7");
+ &movdqa (&QWP(16*5,"esp"),"xmm0");
&mov ("esp",&DWP(16*7+4,"esp")); # restore %esp
&function_end("aesni_xts_encrypt");
&movups (&QWP(0,$out),$inout0); # write output
&set_label("xts_dec_ret");
+ &pxor ("xmm0","xmm0"); # clear register bank
+ &pxor ("xmm1","xmm1");
+ &pxor ("xmm2","xmm2");
+ &movdqa (&QWP(16*0,"esp"),"xmm0"); # clear stack
+ &pxor ("xmm3","xmm3");
+ &movdqa (&QWP(16*1,"esp"),"xmm0");
+ &pxor ("xmm4","xmm4");
+ &movdqa (&QWP(16*2,"esp"),"xmm0");
+ &pxor ("xmm5","xmm5");
+ &movdqa (&QWP(16*3,"esp"),"xmm0");
+ &pxor ("xmm6","xmm6");
+ &movdqa (&QWP(16*4,"esp"),"xmm0");
+ &pxor ("xmm7","xmm7");
+ &movdqa (&QWP(16*5,"esp"),"xmm0");
&mov ("esp",&DWP(16*7+4,"esp")); # restore %esp
&function_end("aesni_xts_decrypt");
}
+\f
+######################################################################
+# void aesni_ocb_[en|de]crypt(const char *inp, char *out, size_t blocks,
+# const AES_KEY *key, unsigned int start_block_num,
+# unsigned char offset_i[16], const unsigned char L_[][16],
+# unsigned char checksum[16]);
+#
+{
+# offsets within stack frame
+my $checksum = 16*6;
+my ($key_off,$rounds_off,$out_off,$end_off,$esp_off)=map(16*7+4*$_,(0..4));
+
+# reassigned registers
+my ($l_,$block,$i1,$i3,$i5) = ($rounds_,$key_,$rounds,$len,$out);
+# $l_, $blocks, $inp, $key are permanently allocated in registers;
+# remaining non-volatile ones are offloaded to stack, which even
+# stay invariant after written to stack.
+
+&function_begin("aesni_ocb_encrypt");
+ &mov ($rounds,&wparam(5)); # &offset_i
+ &mov ($rounds_,&wparam(7)); # &checksum
+
+ &mov ($inp,&wparam(0));
+ &mov ($out,&wparam(1));
+ &mov ($len,&wparam(2));
+ &mov ($key,&wparam(3));
+ &movdqu ($rndkey0,&QWP(0,$rounds)); # load offset_i
+ &mov ($block,&wparam(4)); # start_block_num
+ &movdqu ($rndkey1,&QWP(0,$rounds_)); # load checksum
+ &mov ($l_,&wparam(6)); # L_
+
+ &mov ($rounds,"esp");
+ &sub ("esp",$esp_off+4); # alloca
+ &and ("esp",-16); # align stack
+
+ &sub ($out,$inp);
+ &shl ($len,4);
+ &lea ($len,&DWP(-16*6,$inp,$len)); # end of input - 16*6
+ &mov (&DWP($out_off,"esp"),$out);
+ &mov (&DWP($end_off,"esp"),$len);
+ &mov (&DWP($esp_off,"esp"),$rounds);
+
+ &mov ($rounds,&DWP(240,$key));
+
+ &test ($block,1);
+ &jnz (&label("odd"));
+
+ &bsf ($i3,$block);
+ &add ($block,1);
+ &shl ($i3,4);
+ &movdqu ($inout5,&QWP(0,$l_,$i3));
+ &mov ($i3,$key); # put aside key
+
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &lea ($inp,&DWP(16,$inp));
+
+ &pxor ($inout5,$rndkey0); # ^ last offset_i
+ &pxor ($rndkey1,$inout0); # checksum
+ &pxor ($inout0,$inout5); # ^ offset_i
+
+ &movdqa ($inout4,$rndkey1);
+ if ($inline)
+ { &aesni_inline_generate1("enc"); }
+ else
+ { &call ("_aesni_encrypt1"); }
+
+ &xorps ($inout0,$inout5); # ^ offset_i
+ &movdqa ($rndkey0,$inout5); # pass last offset_i
+ &movdqa ($rndkey1,$inout4); # pass the checksum
+
+ &movups (&QWP(-16,$out,$inp),$inout0); # store output
+
+ &mov ($rounds,&DWP(240,$i3));
+ &mov ($key,$i3); # restore key
+ &mov ($len,&DWP($end_off,"esp"));
+
+&set_label("odd");
+ &shl ($rounds,4);
+ &mov ($out,16);
+ &sub ($out,$rounds); # twisted rounds
+ &mov (&DWP($key_off,"esp"),$key);
+ &lea ($key,&DWP(32,$key,$rounds)); # end of key schedule
+ &mov (&DWP($rounds_off,"esp"),$out);
+
+ &cmp ($inp,$len);
+ &ja (&label("short"));
+ &jmp (&label("grandloop"));
+
+&set_label("grandloop",32);
+ &lea ($i1,&DWP(1,$block));
+ &lea ($i3,&DWP(3,$block));
+ &lea ($i5,&DWP(5,$block));
+ &add ($block,6);
+ &bsf ($i1,$i1);
+ &bsf ($i3,$i3);
+ &bsf ($i5,$i5);
+ &shl ($i1,4);
+ &shl ($i3,4);
+ &shl ($i5,4);
+ &movdqu ($inout0,&QWP(0,$l_));
+ &movdqu ($inout1,&QWP(0,$l_,$i1));
+ &mov ($rounds,&DWP($rounds_off,"esp"));
+ &movdqa ($inout2,$inout0);
+ &movdqu ($inout3,&QWP(0,$l_,$i3));
+ &movdqa ($inout4,$inout0);
+ &movdqu ($inout5,&QWP(0,$l_,$i5));
+
+ &pxor ($inout0,$rndkey0); # ^ last offset_i
+ &pxor ($inout1,$inout0);
+ &movdqa (&QWP(16*0,"esp"),$inout0);
+ &pxor ($inout2,$inout1);
+ &movdqa (&QWP(16*1,"esp"),$inout1);
+ &pxor ($inout3,$inout2);
+ &movdqa (&QWP(16*2,"esp"),$inout2);
+ &pxor ($inout4,$inout3);
+ &movdqa (&QWP(16*3,"esp"),$inout3);
+ &pxor ($inout5,$inout4);
+ &movdqa (&QWP(16*4,"esp"),$inout4);
+ &movdqa (&QWP(16*5,"esp"),$inout5);
+
+ &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &movdqu ($inout1,&QWP(16*1,$inp));
+ &movdqu ($inout2,&QWP(16*2,$inp));
+ &movdqu ($inout3,&QWP(16*3,$inp));
+ &movdqu ($inout4,&QWP(16*4,$inp));
+ &movdqu ($inout5,&QWP(16*5,$inp));
+ &lea ($inp,&DWP(16*6,$inp));
+
+ &pxor ($rndkey1,$inout0); # checksum
+ &pxor ($inout0,$rndkey0); # ^ roundkey[0]
+ &pxor ($rndkey1,$inout1);
+ &pxor ($inout1,$rndkey0);
+ &pxor ($rndkey1,$inout2);
+ &pxor ($inout2,$rndkey0);
+ &pxor ($rndkey1,$inout3);
+ &pxor ($inout3,$rndkey0);
+ &pxor ($rndkey1,$inout4);
+ &pxor ($inout4,$rndkey0);
+ &pxor ($rndkey1,$inout5);
+ &pxor ($inout5,$rndkey0);
+ &movdqa (&QWP($checksum,"esp"),$rndkey1);
+
+ &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
+ &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &pxor ($inout1,&QWP(16*1,"esp"));
+ &pxor ($inout2,&QWP(16*2,"esp"));
+ &pxor ($inout3,&QWP(16*3,"esp"));
+ &pxor ($inout4,&QWP(16*4,"esp"));
+ &pxor ($inout5,&QWP(16*5,"esp"));
+
+ &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
+ &aesenc ($inout0,$rndkey1);
+ &aesenc ($inout1,$rndkey1);
+ &aesenc ($inout2,$rndkey1);
+ &aesenc ($inout3,$rndkey1);
+ &aesenc ($inout4,$rndkey1);
+ &aesenc ($inout5,$rndkey1);
+
+ &mov ($out,&DWP($out_off,"esp"));
+ &mov ($len,&DWP($end_off,"esp"));
+ &call ("_aesni_encrypt6_enter");
+
+ &movdqa ($rndkey0,&QWP(16*5,"esp")); # pass last offset_i
+ &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &pxor ($inout1,&QWP(16*1,"esp"));
+ &pxor ($inout2,&QWP(16*2,"esp"));
+ &pxor ($inout3,&QWP(16*3,"esp"));
+ &pxor ($inout4,&QWP(16*4,"esp"));
+ &pxor ($inout5,$rndkey0);
+ &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
+
+ &movdqu (&QWP(-16*6,$out,$inp),$inout0);# store output
+ &movdqu (&QWP(-16*5,$out,$inp),$inout1);
+ &movdqu (&QWP(-16*4,$out,$inp),$inout2);
+ &movdqu (&QWP(-16*3,$out,$inp),$inout3);
+ &movdqu (&QWP(-16*2,$out,$inp),$inout4);
+ &movdqu (&QWP(-16*1,$out,$inp),$inout5);
+ &cmp ($inp,$len); # done yet?
+ &jb (&label("grandloop"));
+
+&set_label("short");
+ &add ($len,16*6);
+ &sub ($len,$inp);
+ &jz (&label("done"));
+
+ &cmp ($len,16*2);
+ &jb (&label("one"));
+ &je (&label("two"));
+
+ &cmp ($len,16*4);
+ &jb (&label("three"));
+ &je (&label("four"));
+
+ &lea ($i1,&DWP(1,$block));
+ &lea ($i3,&DWP(3,$block));
+ &bsf ($i1,$i1);
+ &bsf ($i3,$i3);
+ &shl ($i1,4);
+ &shl ($i3,4);
+ &movdqu ($inout0,&QWP(0,$l_));
+ &movdqu ($inout1,&QWP(0,$l_,$i1));
+ &mov ($rounds,&DWP($rounds_off,"esp"));
+ &movdqa ($inout2,$inout0);
+ &movdqu ($inout3,&QWP(0,$l_,$i3));
+ &movdqa ($inout4,$inout0);
+
+ &pxor ($inout0,$rndkey0); # ^ last offset_i
+ &pxor ($inout1,$inout0);
+ &movdqa (&QWP(16*0,"esp"),$inout0);
+ &pxor ($inout2,$inout1);
+ &movdqa (&QWP(16*1,"esp"),$inout1);
+ &pxor ($inout3,$inout2);
+ &movdqa (&QWP(16*2,"esp"),$inout2);
+ &pxor ($inout4,$inout3);
+ &movdqa (&QWP(16*3,"esp"),$inout3);
+ &pxor ($inout5,$inout4);
+ &movdqa (&QWP(16*4,"esp"),$inout4);
+
+ &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &movdqu ($inout1,&QWP(16*1,$inp));
+ &movdqu ($inout2,&QWP(16*2,$inp));
+ &movdqu ($inout3,&QWP(16*3,$inp));
+ &movdqu ($inout4,&QWP(16*4,$inp));
+ &pxor ($inout5,$inout5);
+
+ &pxor ($rndkey1,$inout0); # checksum
+ &pxor ($inout0,$rndkey0); # ^ roundkey[0]
+ &pxor ($rndkey1,$inout1);
+ &pxor ($inout1,$rndkey0);
+ &pxor ($rndkey1,$inout2);
+ &pxor ($inout2,$rndkey0);
+ &pxor ($rndkey1,$inout3);
+ &pxor ($inout3,$rndkey0);
+ &pxor ($rndkey1,$inout4);
+ &pxor ($inout4,$rndkey0);
+ &movdqa (&QWP($checksum,"esp"),$rndkey1);
+
+ &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
+ &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &pxor ($inout1,&QWP(16*1,"esp"));
+ &pxor ($inout2,&QWP(16*2,"esp"));
+ &pxor ($inout3,&QWP(16*3,"esp"));
+ &pxor ($inout4,&QWP(16*4,"esp"));
+
+ &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
+ &aesenc ($inout0,$rndkey1);
+ &aesenc ($inout1,$rndkey1);
+ &aesenc ($inout2,$rndkey1);
+ &aesenc ($inout3,$rndkey1);
+ &aesenc ($inout4,$rndkey1);
+ &aesenc ($inout5,$rndkey1);
+
+ &mov ($out,&DWP($out_off,"esp"));
+ &call ("_aesni_encrypt6_enter");
+
+ &movdqa ($rndkey0,&QWP(16*4,"esp")); # pass last offset_i
+ &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &pxor ($inout1,&QWP(16*1,"esp"));
+ &pxor ($inout2,&QWP(16*2,"esp"));
+ &pxor ($inout3,&QWP(16*3,"esp"));
+ &pxor ($inout4,$rndkey0);
+ &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
+
+ &movdqu (&QWP(16*0,$out,$inp),$inout0); # store output
+ &movdqu (&QWP(16*1,$out,$inp),$inout1);
+ &movdqu (&QWP(16*2,$out,$inp),$inout2);
+ &movdqu (&QWP(16*3,$out,$inp),$inout3);
+ &movdqu (&QWP(16*4,$out,$inp),$inout4);
+
+ &jmp (&label("done"));
+
+&set_label("one",16);
+ &movdqu ($inout5,&QWP(0,$l_));
+ &mov ($key,&DWP($key_off,"esp")); # restore key
+
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &mov ($rounds,&DWP(240,$key));
+
+ &pxor ($inout5,$rndkey0); # ^ last offset_i
+ &pxor ($rndkey1,$inout0); # checksum
+ &pxor ($inout0,$inout5); # ^ offset_i
+
+ &movdqa ($inout4,$rndkey1);
+ &mov ($out,&DWP($out_off,"esp"));
+ if ($inline)
+ { &aesni_inline_generate1("enc"); }
+ else
+ { &call ("_aesni_encrypt1"); }
+
+ &xorps ($inout0,$inout5); # ^ offset_i
+ &movdqa ($rndkey0,$inout5); # pass last offset_i
+ &movdqa ($rndkey1,$inout4); # pass the checksum
+ &movups (&QWP(0,$out,$inp),$inout0);
+
+ &jmp (&label("done"));
+
+&set_label("two",16);
+ &lea ($i1,&DWP(1,$block));
+ &mov ($key,&DWP($key_off,"esp")); # restore key
+ &bsf ($i1,$i1);
+ &shl ($i1,4);
+ &movdqu ($inout4,&QWP(0,$l_));
+ &movdqu ($inout5,&QWP(0,$l_,$i1));
+
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &movdqu ($inout1,&QWP(16*1,$inp));
+ &mov ($rounds,&DWP(240,$key));
+
+ &pxor ($inout4,$rndkey0); # ^ last offset_i
+ &pxor ($inout5,$inout4);
+
+ &pxor ($rndkey1,$inout0); # checksum
+ &pxor ($inout0,$inout4); # ^ offset_i
+ &pxor ($rndkey1,$inout1);
+ &pxor ($inout1,$inout5);
+
+ &movdqa ($inout3,$rndkey1)
+ &mov ($out,&DWP($out_off,"esp"));
+ &call ("_aesni_encrypt2");
+
+ &xorps ($inout0,$inout4); # ^ offset_i
+ &xorps ($inout1,$inout5);
+ &movdqa ($rndkey0,$inout5); # pass last offset_i
+ &movdqa ($rndkey1,$inout3); # pass the checksum
+ &movups (&QWP(16*0,$out,$inp),$inout0); # store output
+ &movups (&QWP(16*1,$out,$inp),$inout1);
+
+ &jmp (&label("done"));
+
+&set_label("three",16);
+ &lea ($i1,&DWP(1,$block));
+ &mov ($key,&DWP($key_off,"esp")); # restore key
+ &bsf ($i1,$i1);
+ &shl ($i1,4);
+ &movdqu ($inout3,&QWP(0,$l_));
+ &movdqu ($inout4,&QWP(0,$l_,$i1));
+ &movdqa ($inout5,$inout3);
+
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &movdqu ($inout1,&QWP(16*1,$inp));
+ &movdqu ($inout2,&QWP(16*2,$inp));
+ &mov ($rounds,&DWP(240,$key));
+
+ &pxor ($inout3,$rndkey0); # ^ last offset_i
+ &pxor ($inout4,$inout3);
+ &pxor ($inout5,$inout4);
+
+ &pxor ($rndkey1,$inout0); # checksum
+ &pxor ($inout0,$inout3); # ^ offset_i
+ &pxor ($rndkey1,$inout1);
+ &pxor ($inout1,$inout4);
+ &pxor ($rndkey1,$inout2);
+ &pxor ($inout2,$inout5);
+
+ &movdqa (&QWP($checksum,"esp"),$rndkey1);
+ &mov ($out,&DWP($out_off,"esp"));
+ &call ("_aesni_encrypt3");
+
+ &xorps ($inout0,$inout3); # ^ offset_i
+ &xorps ($inout1,$inout4);
+ &xorps ($inout2,$inout5);
+ &movdqa ($rndkey0,$inout5); # pass last offset_i
+ &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
+ &movups (&QWP(16*0,$out,$inp),$inout0); # store output
+ &movups (&QWP(16*1,$out,$inp),$inout1);
+ &movups (&QWP(16*2,$out,$inp),$inout2);
+
+ &jmp (&label("done"));
+
+&set_label("four",16);
+ &lea ($i1,&DWP(1,$block));
+ &lea ($i3,&DWP(3,$block));
+ &bsf ($i1,$i1);
+ &bsf ($i3,$i3);
+ &mov ($key,&DWP($key_off,"esp")); # restore key
+ &shl ($i1,4);
+ &shl ($i3,4);
+ &movdqu ($inout2,&QWP(0,$l_));
+ &movdqu ($inout3,&QWP(0,$l_,$i1));
+ &movdqa ($inout4,$inout2);
+ &movdqu ($inout5,&QWP(0,$l_,$i3));
+
+ &pxor ($inout2,$rndkey0); # ^ last offset_i
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &pxor ($inout3,$inout2);
+ &movdqu ($inout1,&QWP(16*1,$inp));
+ &pxor ($inout4,$inout3);
+ &movdqa (&QWP(16*0,"esp"),$inout2);
+ &pxor ($inout5,$inout4);
+ &movdqa (&QWP(16*1,"esp"),$inout3);
+ &movdqu ($inout2,&QWP(16*2,$inp));
+ &movdqu ($inout3,&QWP(16*3,$inp));
+ &mov ($rounds,&DWP(240,$key));
+
+ &pxor ($rndkey1,$inout0); # checksum
+ &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &pxor ($rndkey1,$inout1);
+ &pxor ($inout1,&QWP(16*1,"esp"));
+ &pxor ($rndkey1,$inout2);
+ &pxor ($inout2,$inout4);
+ &pxor ($rndkey1,$inout3);
+ &pxor ($inout3,$inout5);
+
+ &movdqa (&QWP($checksum,"esp"),$rndkey1)
+ &mov ($out,&DWP($out_off,"esp"));
+ &call ("_aesni_encrypt4");
+
+ &xorps ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &xorps ($inout1,&QWP(16*1,"esp"));
+ &xorps ($inout2,$inout4);
+ &movups (&QWP(16*0,$out,$inp),$inout0); # store output
+ &xorps ($inout3,$inout5);
+ &movups (&QWP(16*1,$out,$inp),$inout1);
+ &movdqa ($rndkey0,$inout5); # pass last offset_i
+ &movups (&QWP(16*2,$out,$inp),$inout2);
+ &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
+ &movups (&QWP(16*3,$out,$inp),$inout3);
+
+&set_label("done");
+ &mov ($key,&DWP($esp_off,"esp"));
+ &pxor ($inout0,$inout0); # clear register bank
+ &pxor ($inout1,$inout1);
+ &movdqa (&QWP(16*0,"esp"),$inout0); # clear stack
+ &pxor ($inout2,$inout2);
+ &movdqa (&QWP(16*1,"esp"),$inout0);
+ &pxor ($inout3,$inout3);
+ &movdqa (&QWP(16*2,"esp"),$inout0);
+ &pxor ($inout4,$inout4);
+ &movdqa (&QWP(16*3,"esp"),$inout0);
+ &pxor ($inout5,$inout5);
+ &movdqa (&QWP(16*4,"esp"),$inout0);
+ &movdqa (&QWP(16*5,"esp"),$inout0);
+ &movdqa (&QWP(16*6,"esp"),$inout0);
+
+ &lea ("esp",&DWP(0,$key));
+ &mov ($rounds,&wparam(5)); # &offset_i
+ &mov ($rounds_,&wparam(7)); # &checksum
+ &movdqu (&QWP(0,$rounds),$rndkey0);
+ &pxor ($rndkey0,$rndkey0);
+ &movdqu (&QWP(0,$rounds_),$rndkey1);
+ &pxor ($rndkey1,$rndkey1);
+&function_end("aesni_ocb_encrypt");
+
+&function_begin("aesni_ocb_decrypt");
+ &mov ($rounds,&wparam(5)); # &offset_i
+ &mov ($rounds_,&wparam(7)); # &checksum
+
+ &mov ($inp,&wparam(0));
+ &mov ($out,&wparam(1));
+ &mov ($len,&wparam(2));
+ &mov ($key,&wparam(3));
+ &movdqu ($rndkey0,&QWP(0,$rounds)); # load offset_i
+ &mov ($block,&wparam(4)); # start_block_num
+ &movdqu ($rndkey1,&QWP(0,$rounds_)); # load checksum
+ &mov ($l_,&wparam(6)); # L_
+
+ &mov ($rounds,"esp");
+ &sub ("esp",$esp_off+4); # alloca
+ &and ("esp",-16); # align stack
+
+ &sub ($out,$inp);
+ &shl ($len,4);
+ &lea ($len,&DWP(-16*6,$inp,$len)); # end of input - 16*6
+ &mov (&DWP($out_off,"esp"),$out);
+ &mov (&DWP($end_off,"esp"),$len);
+ &mov (&DWP($esp_off,"esp"),$rounds);
+
+ &mov ($rounds,&DWP(240,$key));
+
+ &test ($block,1);
+ &jnz (&label("odd"));
+
+ &bsf ($i3,$block);
+ &add ($block,1);
+ &shl ($i3,4);
+ &movdqu ($inout5,&QWP(0,$l_,$i3));
+ &mov ($i3,$key); # put aside key
+
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &lea ($inp,&DWP(16,$inp));
+
+ &pxor ($inout5,$rndkey0); # ^ last offset_i
+ &pxor ($inout0,$inout5); # ^ offset_i
+
+ &movdqa ($inout4,$rndkey1);
+ if ($inline)
+ { &aesni_inline_generate1("dec"); }
+ else
+ { &call ("_aesni_decrypt1"); }
+
+ &xorps ($inout0,$inout5); # ^ offset_i
+ &movaps ($rndkey1,$inout4); # pass the checksum
+ &movdqa ($rndkey0,$inout5); # pass last offset_i
+ &xorps ($rndkey1,$inout0); # checksum
+ &movups (&QWP(-16,$out,$inp),$inout0); # store output
+
+ &mov ($rounds,&DWP(240,$i3));
+ &mov ($key,$i3); # restore key
+ &mov ($len,&DWP($end_off,"esp"));
+
+&set_label("odd");
+ &shl ($rounds,4);
+ &mov ($out,16);
+ &sub ($out,$rounds); # twisted rounds
+ &mov (&DWP($key_off,"esp"),$key);
+ &lea ($key,&DWP(32,$key,$rounds)); # end of key schedule
+ &mov (&DWP($rounds_off,"esp"),$out);
+
+ &cmp ($inp,$len);
+ &ja (&label("short"));
+ &jmp (&label("grandloop"));
+
+&set_label("grandloop",32);
+ &lea ($i1,&DWP(1,$block));
+ &lea ($i3,&DWP(3,$block));
+ &lea ($i5,&DWP(5,$block));
+ &add ($block,6);
+ &bsf ($i1,$i1);
+ &bsf ($i3,$i3);
+ &bsf ($i5,$i5);
+ &shl ($i1,4);
+ &shl ($i3,4);
+ &shl ($i5,4);
+ &movdqu ($inout0,&QWP(0,$l_));
+ &movdqu ($inout1,&QWP(0,$l_,$i1));
+ &mov ($rounds,&DWP($rounds_off,"esp"));
+ &movdqa ($inout2,$inout0);
+ &movdqu ($inout3,&QWP(0,$l_,$i3));
+ &movdqa ($inout4,$inout0);
+ &movdqu ($inout5,&QWP(0,$l_,$i5));
+
+ &pxor ($inout0,$rndkey0); # ^ last offset_i
+ &pxor ($inout1,$inout0);
+ &movdqa (&QWP(16*0,"esp"),$inout0);
+ &pxor ($inout2,$inout1);
+ &movdqa (&QWP(16*1,"esp"),$inout1);
+ &pxor ($inout3,$inout2);
+ &movdqa (&QWP(16*2,"esp"),$inout2);
+ &pxor ($inout4,$inout3);
+ &movdqa (&QWP(16*3,"esp"),$inout3);
+ &pxor ($inout5,$inout4);
+ &movdqa (&QWP(16*4,"esp"),$inout4);
+ &movdqa (&QWP(16*5,"esp"),$inout5);
+
+ &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &movdqu ($inout1,&QWP(16*1,$inp));
+ &movdqu ($inout2,&QWP(16*2,$inp));
+ &movdqu ($inout3,&QWP(16*3,$inp));
+ &movdqu ($inout4,&QWP(16*4,$inp));
+ &movdqu ($inout5,&QWP(16*5,$inp));
+ &lea ($inp,&DWP(16*6,$inp));
+
+ &movdqa (&QWP($checksum,"esp"),$rndkey1);
+ &pxor ($inout0,$rndkey0); # ^ roundkey[0]
+ &pxor ($inout1,$rndkey0);
+ &pxor ($inout2,$rndkey0);
+ &pxor ($inout3,$rndkey0);
+ &pxor ($inout4,$rndkey0);
+ &pxor ($inout5,$rndkey0);
+
+ &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
+ &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &pxor ($inout1,&QWP(16*1,"esp"));
+ &pxor ($inout2,&QWP(16*2,"esp"));
+ &pxor ($inout3,&QWP(16*3,"esp"));
+ &pxor ($inout4,&QWP(16*4,"esp"));
+ &pxor ($inout5,&QWP(16*5,"esp"));
+
+ &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
+ &aesdec ($inout0,$rndkey1);
+ &aesdec ($inout1,$rndkey1);
+ &aesdec ($inout2,$rndkey1);
+ &aesdec ($inout3,$rndkey1);
+ &aesdec ($inout4,$rndkey1);
+ &aesdec ($inout5,$rndkey1);
+
+ &mov ($out,&DWP($out_off,"esp"));
+ &mov ($len,&DWP($end_off,"esp"));
+ &call ("_aesni_decrypt6_enter");
+
+ &movdqa ($rndkey0,&QWP(16*5,"esp")); # pass last offset_i
+ &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &movdqa ($rndkey1,&QWP($checksum,"esp"));
+ &pxor ($inout1,&QWP(16*1,"esp"));
+ &pxor ($inout2,&QWP(16*2,"esp"));
+ &pxor ($inout3,&QWP(16*3,"esp"));
+ &pxor ($inout4,&QWP(16*4,"esp"));
+ &pxor ($inout5,$rndkey0);
+
+ &pxor ($rndkey1,$inout0); # checksum
+ &movdqu (&QWP(-16*6,$out,$inp),$inout0);# store output
+ &pxor ($rndkey1,$inout1);
+ &movdqu (&QWP(-16*5,$out,$inp),$inout1);
+ &pxor ($rndkey1,$inout2);
+ &movdqu (&QWP(-16*4,$out,$inp),$inout2);
+ &pxor ($rndkey1,$inout3);
+ &movdqu (&QWP(-16*3,$out,$inp),$inout3);
+ &pxor ($rndkey1,$inout4);
+ &movdqu (&QWP(-16*2,$out,$inp),$inout4);
+ &pxor ($rndkey1,$inout5);
+ &movdqu (&QWP(-16*1,$out,$inp),$inout5);
+ &cmp ($inp,$len); # done yet?
+ &jb (&label("grandloop"));
+
+&set_label("short");
+ &add ($len,16*6);
+ &sub ($len,$inp);
+ &jz (&label("done"));
+
+ &cmp ($len,16*2);
+ &jb (&label("one"));
+ &je (&label("two"));
+
+ &cmp ($len,16*4);
+ &jb (&label("three"));
+ &je (&label("four"));
+
+ &lea ($i1,&DWP(1,$block));
+ &lea ($i3,&DWP(3,$block));
+ &bsf ($i1,$i1);
+ &bsf ($i3,$i3);
+ &shl ($i1,4);
+ &shl ($i3,4);
+ &movdqu ($inout0,&QWP(0,$l_));
+ &movdqu ($inout1,&QWP(0,$l_,$i1));
+ &mov ($rounds,&DWP($rounds_off,"esp"));
+ &movdqa ($inout2,$inout0);
+ &movdqu ($inout3,&QWP(0,$l_,$i3));
+ &movdqa ($inout4,$inout0);
+
+ &pxor ($inout0,$rndkey0); # ^ last offset_i
+ &pxor ($inout1,$inout0);
+ &movdqa (&QWP(16*0,"esp"),$inout0);
+ &pxor ($inout2,$inout1);
+ &movdqa (&QWP(16*1,"esp"),$inout1);
+ &pxor ($inout3,$inout2);
+ &movdqa (&QWP(16*2,"esp"),$inout2);
+ &pxor ($inout4,$inout3);
+ &movdqa (&QWP(16*3,"esp"),$inout3);
+ &pxor ($inout5,$inout4);
+ &movdqa (&QWP(16*4,"esp"),$inout4);
+
+ &$movekey ($rndkey0,&QWP(-48,$key,$rounds));
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &movdqu ($inout1,&QWP(16*1,$inp));
+ &movdqu ($inout2,&QWP(16*2,$inp));
+ &movdqu ($inout3,&QWP(16*3,$inp));
+ &movdqu ($inout4,&QWP(16*4,$inp));
+ &pxor ($inout5,$inout5);
+
+ &movdqa (&QWP($checksum,"esp"),$rndkey1);
+ &pxor ($inout0,$rndkey0); # ^ roundkey[0]
+ &pxor ($inout1,$rndkey0);
+ &pxor ($inout2,$rndkey0);
+ &pxor ($inout3,$rndkey0);
+ &pxor ($inout4,$rndkey0);
+
+ &$movekey ($rndkey1,&QWP(-32,$key,$rounds));
+ &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &pxor ($inout1,&QWP(16*1,"esp"));
+ &pxor ($inout2,&QWP(16*2,"esp"));
+ &pxor ($inout3,&QWP(16*3,"esp"));
+ &pxor ($inout4,&QWP(16*4,"esp"));
+
+ &$movekey ($rndkey0,&QWP(-16,$key,$rounds));
+ &aesdec ($inout0,$rndkey1);
+ &aesdec ($inout1,$rndkey1);
+ &aesdec ($inout2,$rndkey1);
+ &aesdec ($inout3,$rndkey1);
+ &aesdec ($inout4,$rndkey1);
+ &aesdec ($inout5,$rndkey1);
+
+ &mov ($out,&DWP($out_off,"esp"));
+ &call ("_aesni_decrypt6_enter");
+
+ &movdqa ($rndkey0,&QWP(16*4,"esp")); # pass last offset_i
+ &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &movdqa ($rndkey1,&QWP($checksum,"esp"));
+ &pxor ($inout1,&QWP(16*1,"esp"));
+ &pxor ($inout2,&QWP(16*2,"esp"));
+ &pxor ($inout3,&QWP(16*3,"esp"));
+ &pxor ($inout4,$rndkey0);
+
+ &pxor ($rndkey1,$inout0); # checksum
+ &movdqu (&QWP(16*0,$out,$inp),$inout0); # store output
+ &pxor ($rndkey1,$inout1);
+ &movdqu (&QWP(16*1,$out,$inp),$inout1);
+ &pxor ($rndkey1,$inout2);
+ &movdqu (&QWP(16*2,$out,$inp),$inout2);
+ &pxor ($rndkey1,$inout3);
+ &movdqu (&QWP(16*3,$out,$inp),$inout3);
+ &pxor ($rndkey1,$inout4);
+ &movdqu (&QWP(16*4,$out,$inp),$inout4);
+
+ &jmp (&label("done"));
+
+&set_label("one",16);
+ &movdqu ($inout5,&QWP(0,$l_));
+ &mov ($key,&DWP($key_off,"esp")); # restore key
+
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &mov ($rounds,&DWP(240,$key));
+
+ &pxor ($inout5,$rndkey0); # ^ last offset_i
+ &pxor ($inout0,$inout5); # ^ offset_i
+
+ &movdqa ($inout4,$rndkey1);
+ &mov ($out,&DWP($out_off,"esp"));
+ if ($inline)
+ { &aesni_inline_generate1("dec"); }
+ else
+ { &call ("_aesni_decrypt1"); }
+
+ &xorps ($inout0,$inout5); # ^ offset_i
+ &movaps ($rndkey1,$inout4); # pass the checksum
+ &movdqa ($rndkey0,$inout5); # pass last offset_i
+ &xorps ($rndkey1,$inout0); # checksum
+ &movups (&QWP(0,$out,$inp),$inout0);
+
+ &jmp (&label("done"));
+
+&set_label("two",16);
+ &lea ($i1,&DWP(1,$block));
+ &mov ($key,&DWP($key_off,"esp")); # restore key
+ &bsf ($i1,$i1);
+ &shl ($i1,4);
+ &movdqu ($inout4,&QWP(0,$l_));
+ &movdqu ($inout5,&QWP(0,$l_,$i1));
+
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &movdqu ($inout1,&QWP(16*1,$inp));
+ &mov ($rounds,&DWP(240,$key));
+
+ &movdqa ($inout3,$rndkey1);
+ &pxor ($inout4,$rndkey0); # ^ last offset_i
+ &pxor ($inout5,$inout4);
+
+ &pxor ($inout0,$inout4); # ^ offset_i
+ &pxor ($inout1,$inout5);
+
+ &mov ($out,&DWP($out_off,"esp"));
+ &call ("_aesni_decrypt2");
+
+ &xorps ($inout0,$inout4); # ^ offset_i
+ &xorps ($inout1,$inout5);
+ &movdqa ($rndkey0,$inout5); # pass last offset_i
+ &xorps ($inout3,$inout0); # checksum
+ &movups (&QWP(16*0,$out,$inp),$inout0); # store output
+ &xorps ($inout3,$inout1);
+ &movups (&QWP(16*1,$out,$inp),$inout1);
+ &movaps ($rndkey1,$inout3); # pass the checksum
+
+ &jmp (&label("done"));
+
+&set_label("three",16);
+ &lea ($i1,&DWP(1,$block));
+ &mov ($key,&DWP($key_off,"esp")); # restore key
+ &bsf ($i1,$i1);
+ &shl ($i1,4);
+ &movdqu ($inout3,&QWP(0,$l_));
+ &movdqu ($inout4,&QWP(0,$l_,$i1));
+ &movdqa ($inout5,$inout3);
+
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &movdqu ($inout1,&QWP(16*1,$inp));
+ &movdqu ($inout2,&QWP(16*2,$inp));
+ &mov ($rounds,&DWP(240,$key));
+
+ &movdqa (&QWP($checksum,"esp"),$rndkey1);
+ &pxor ($inout3,$rndkey0); # ^ last offset_i
+ &pxor ($inout4,$inout3);
+ &pxor ($inout5,$inout4);
+
+ &pxor ($inout0,$inout3); # ^ offset_i
+ &pxor ($inout1,$inout4);
+ &pxor ($inout2,$inout5);
+
+ &mov ($out,&DWP($out_off,"esp"));
+ &call ("_aesni_decrypt3");
+
+ &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
+ &xorps ($inout0,$inout3); # ^ offset_i
+ &xorps ($inout1,$inout4);
+ &xorps ($inout2,$inout5);
+ &movups (&QWP(16*0,$out,$inp),$inout0); # store output
+ &pxor ($rndkey1,$inout0); # checksum
+ &movdqa ($rndkey0,$inout5); # pass last offset_i
+ &movups (&QWP(16*1,$out,$inp),$inout1);
+ &pxor ($rndkey1,$inout1);
+ &movups (&QWP(16*2,$out,$inp),$inout2);
+ &pxor ($rndkey1,$inout2);
+
+ &jmp (&label("done"));
+
+&set_label("four",16);
+ &lea ($i1,&DWP(1,$block));
+ &lea ($i3,&DWP(3,$block));
+ &bsf ($i1,$i1);
+ &bsf ($i3,$i3);
+ &mov ($key,&DWP($key_off,"esp")); # restore key
+ &shl ($i1,4);
+ &shl ($i3,4);
+ &movdqu ($inout2,&QWP(0,$l_));
+ &movdqu ($inout3,&QWP(0,$l_,$i1));
+ &movdqa ($inout4,$inout2);
+ &movdqu ($inout5,&QWP(0,$l_,$i3));
+
+ &pxor ($inout2,$rndkey0); # ^ last offset_i
+ &movdqu ($inout0,&QWP(16*0,$inp)); # load input
+ &pxor ($inout3,$inout2);
+ &movdqu ($inout1,&QWP(16*1,$inp));
+ &pxor ($inout4,$inout3);
+ &movdqa (&QWP(16*0,"esp"),$inout2);
+ &pxor ($inout5,$inout4);
+ &movdqa (&QWP(16*1,"esp"),$inout3);
+ &movdqu ($inout2,&QWP(16*2,$inp));
+ &movdqu ($inout3,&QWP(16*3,$inp));
+ &mov ($rounds,&DWP(240,$key));
+
+ &movdqa (&QWP($checksum,"esp"),$rndkey1);
+ &pxor ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &pxor ($inout1,&QWP(16*1,"esp"));
+ &pxor ($inout2,$inout4);
+ &pxor ($inout3,$inout5);
+
+ &mov ($out,&DWP($out_off,"esp"));
+ &call ("_aesni_decrypt4");
+
+ &movdqa ($rndkey1,&QWP($checksum,"esp"));# pass the checksum
+ &xorps ($inout0,&QWP(16*0,"esp")); # ^ offset_i
+ &xorps ($inout1,&QWP(16*1,"esp"));
+ &xorps ($inout2,$inout4);
+ &movups (&QWP(16*0,$out,$inp),$inout0); # store output
+ &pxor ($rndkey1,$inout0); # checksum
+ &xorps ($inout3,$inout5);
+ &movups (&QWP(16*1,$out,$inp),$inout1);
+ &pxor ($rndkey1,$inout1);
+ &movdqa ($rndkey0,$inout5); # pass last offset_i
+ &movups (&QWP(16*2,$out,$inp),$inout2);
+ &pxor ($rndkey1,$inout2);
+ &movups (&QWP(16*3,$out,$inp),$inout3);
+ &pxor ($rndkey1,$inout3);
+
+&set_label("done");
+ &mov ($key,&DWP($esp_off,"esp"));
+ &pxor ($inout0,$inout0); # clear register bank
+ &pxor ($inout1,$inout1);
+ &movdqa (&QWP(16*0,"esp"),$inout0); # clear stack
+ &pxor ($inout2,$inout2);
+ &movdqa (&QWP(16*1,"esp"),$inout0);
+ &pxor ($inout3,$inout3);
+ &movdqa (&QWP(16*2,"esp"),$inout0);
+ &pxor ($inout4,$inout4);
+ &movdqa (&QWP(16*3,"esp"),$inout0);
+ &pxor ($inout5,$inout5);
+ &movdqa (&QWP(16*4,"esp"),$inout0);
+ &movdqa (&QWP(16*5,"esp"),$inout0);
+ &movdqa (&QWP(16*6,"esp"),$inout0);
+
+ &lea ("esp",&DWP(0,$key));
+ &mov ($rounds,&wparam(5)); # &offset_i
+ &mov ($rounds_,&wparam(7)); # &checksum
+ &movdqu (&QWP(0,$rounds),$rndkey0);
+ &pxor ($rndkey0,$rndkey0);
+ &movdqu (&QWP(0,$rounds_),$rndkey1);
+ &pxor ($rndkey1,$rndkey1);
+&function_end("aesni_ocb_decrypt");
+}
}
\f
######################################################################
&add ($len,16);
&jnz (&label("cbc_enc_tail"));
&movaps ($ivec,$inout0);
+ &pxor ($inout0,$inout0);
&jmp (&label("cbc_ret"));
&set_label("cbc_enc_tail");
&movaps ($inout0,$inout5);
&movaps ($ivec,$rndkey0);
&add ($len,0x50);
- &jle (&label("cbc_dec_tail_collected"));
+ &jle (&label("cbc_dec_clear_tail_collected"));
&movups (&QWP(0,$out),$inout0);
&lea ($out,&DWP(0x10,$out));
&set_label("cbc_dec_tail");
&xorps ($inout4,$rndkey0);
&movups (&QWP(0,$out),$inout0);
&movups (&QWP(0x10,$out),$inout1);
+ &pxor ($inout1,$inout1);
&movups (&QWP(0x20,$out),$inout2);
+ &pxor ($inout2,$inout2);
&movups (&QWP(0x30,$out),$inout3);
+ &pxor ($inout3,$inout3);
&lea ($out,&DWP(0x40,$out));
&movaps ($inout0,$inout4);
+ &pxor ($inout4,$inout4);
&sub ($len,0x50);
&jmp (&label("cbc_dec_tail_collected"));
&xorps ($inout1,$in0);
&movups (&QWP(0,$out),$inout0);
&movaps ($inout0,$inout1);
+ &pxor ($inout1,$inout1);
&lea ($out,&DWP(0x10,$out));
&movaps ($ivec,$in1);
&sub ($len,0x20);
&xorps ($inout2,$in1);
&movups (&QWP(0,$out),$inout0);
&movaps ($inout0,$inout2);
+ &pxor ($inout2,$inout2);
&movups (&QWP(0x10,$out),$inout1);
+ &pxor ($inout1,$inout1);
&lea ($out,&DWP(0x20,$out));
&movups ($ivec,&QWP(0x20,$inp));
&sub ($len,0x30);
&movups (&QWP(0,$out),$inout0);
&xorps ($inout2,$rndkey1);
&movups (&QWP(0x10,$out),$inout1);
+ &pxor ($inout1,$inout1);
&xorps ($inout3,$rndkey0);
&movups (&QWP(0x20,$out),$inout2);
+ &pxor ($inout2,$inout2);
&lea ($out,&DWP(0x30,$out));
&movaps ($inout0,$inout3);
+ &pxor ($inout3,$inout3);
&sub ($len,0x40);
+ &jmp (&label("cbc_dec_tail_collected"));
+&set_label("cbc_dec_clear_tail_collected",16);
+ &pxor ($inout1,$inout1);
+ &pxor ($inout2,$inout2);
+ &pxor ($inout3,$inout3);
+ &pxor ($inout4,$inout4);
&set_label("cbc_dec_tail_collected");
&and ($len,15);
&jnz (&label("cbc_dec_tail_partial"));
&movups (&QWP(0,$out),$inout0);
+ &pxor ($rndkey0,$rndkey0);
&jmp (&label("cbc_ret"));
&set_label("cbc_dec_tail_partial",16);
&movaps (&QWP(0,"esp"),$inout0);
+ &pxor ($rndkey0,$rndkey0);
&mov ("ecx",16);
&mov ($inp,"esp");
&sub ("ecx",$len);
&data_word(0xA4F3F689); # rep movsb
+ &movdqa (&QWP(0,"esp"),$inout0);
&set_label("cbc_ret");
&mov ("esp",&DWP(16,"esp")); # pull original %esp
&mov ($key_,&wparam(4));
+ &pxor ($inout0,$inout0);
+ &pxor ($rndkey1,$rndkey1);
&movups (&QWP(0,$key_),$ivec); # output IV
+ &pxor ($ivec,$ivec);
&set_label("cbc_abort");
&function_end("${PREFIX}_cbc_encrypt");
\f
# $round rounds
&function_begin_B("_aesni_set_encrypt_key");
+ &push ("ebp");
+ &push ("ebx");
&test ("eax","eax");
&jz (&label("bad_pointer"));
&test ($key,$key);
&jz (&label("bad_pointer"));
+ &call (&label("pic"));
+&set_label("pic");
+ &blindpop("ebx");
+ &lea ("ebx",&DWP(&label("key_const")."-".&label("pic"),"ebx"));
+
+ &picmeup("ebp","OPENSSL_ia32cap_P","ebx",&label("key_const"));
&movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey
&xorps ("xmm4","xmm4"); # low dword of xmm4 is assumed 0
+ &mov ("ebp",&DWP(4,"ebp"));
&lea ($key,&DWP(16,$key));
+ &and ("ebp",1<<28|1<<11); # AVX and XOP bits
&cmp ($rounds,256);
&je (&label("14rounds"));
&cmp ($rounds,192);
&jne (&label("bad_keybits"));
&set_label("10rounds",16);
+ &cmp ("ebp",1<<28);
+ &je (&label("10rounds_alt"));
+
&mov ($rounds,9);
&$movekey (&QWP(-16,$key),"xmm0"); # round 0
&aeskeygenassist("xmm1","xmm0",0x01); # round 1
&call (&label("key_128"));
&$movekey (&QWP(0,$key),"xmm0");
&mov (&DWP(80,$key),$rounds);
- &xor ("eax","eax");
- &ret();
+
+ &jmp (&label("good_key"));
&set_label("key_128",16);
&$movekey (&QWP(0,$key),"xmm0");
&xorps ("xmm0","xmm1");
&ret();
+&set_label("10rounds_alt",16);
+ &movdqa ("xmm5",&QWP(0x00,"ebx"));
+ &mov ($rounds,8);
+ &movdqa ("xmm4",&QWP(0x20,"ebx"));
+ &movdqa ("xmm2","xmm0");
+ &movdqu (&QWP(-16,$key),"xmm0");
+
+&set_label("loop_key128");
+ &pshufb ("xmm0","xmm5");
+ &aesenclast ("xmm0","xmm4");
+ &pslld ("xmm4",1);
+ &lea ($key,&DWP(16,$key));
+
+ &movdqa ("xmm3","xmm2");
+ &pslldq ("xmm2",4);
+ &pxor ("xmm3","xmm2");
+ &pslldq ("xmm2",4);
+ &pxor ("xmm3","xmm2");
+ &pslldq ("xmm2",4);
+ &pxor ("xmm2","xmm3");
+
+ &pxor ("xmm0","xmm2");
+ &movdqu (&QWP(-16,$key),"xmm0");
+ &movdqa ("xmm2","xmm0");
+
+ &dec ($rounds);
+ &jnz (&label("loop_key128"));
+
+ &movdqa ("xmm4",&QWP(0x30,"ebx"));
+
+ &pshufb ("xmm0","xmm5");
+ &aesenclast ("xmm0","xmm4");
+ &pslld ("xmm4",1);
+
+ &movdqa ("xmm3","xmm2");
+ &pslldq ("xmm2",4);
+ &pxor ("xmm3","xmm2");
+ &pslldq ("xmm2",4);
+ &pxor ("xmm3","xmm2");
+ &pslldq ("xmm2",4);
+ &pxor ("xmm2","xmm3");
+
+ &pxor ("xmm0","xmm2");
+ &movdqu (&QWP(0,$key),"xmm0");
+
+ &movdqa ("xmm2","xmm0");
+ &pshufb ("xmm0","xmm5");
+ &aesenclast ("xmm0","xmm4");
+
+ &movdqa ("xmm3","xmm2");
+ &pslldq ("xmm2",4);
+ &pxor ("xmm3","xmm2");
+ &pslldq ("xmm2",4);
+ &pxor ("xmm3","xmm2");
+ &pslldq ("xmm2",4);
+ &pxor ("xmm2","xmm3");
+
+ &pxor ("xmm0","xmm2");
+ &movdqu (&QWP(16,$key),"xmm0");
+
+ &mov ($rounds,9);
+ &mov (&DWP(96,$key),$rounds);
+
+ &jmp (&label("good_key"));
+
&set_label("12rounds",16);
&movq ("xmm2",&QWP(16,"eax")); # remaining 1/3 of *userKey
+ &cmp ("ebp",1<<28);
+ &je (&label("12rounds_alt"));
+
&mov ($rounds,11);
&$movekey (&QWP(-16,$key),"xmm0"); # round 0
&aeskeygenassist("xmm1","xmm2",0x01); # round 1,2
&call (&label("key_192b"));
&$movekey (&QWP(0,$key),"xmm0");
&mov (&DWP(48,$key),$rounds);
- &xor ("eax","eax");
- &ret();
+
+ &jmp (&label("good_key"));
&set_label("key_192a",16);
&$movekey (&QWP(0,$key),"xmm0");
&lea ($key,&DWP(32,$key));
&jmp (&label("key_192b_warm"));
+&set_label("12rounds_alt",16);
+ &movdqa ("xmm5",&QWP(0x10,"ebx"));
+ &movdqa ("xmm4",&QWP(0x20,"ebx"));
+ &mov ($rounds,8);
+ &movdqu (&QWP(-16,$key),"xmm0");
+
+&set_label("loop_key192");
+ &movq (&QWP(0,$key),"xmm2");
+ &movdqa ("xmm1","xmm2");
+ &pshufb ("xmm2","xmm5");
+ &aesenclast ("xmm2","xmm4");
+ &pslld ("xmm4",1);
+ &lea ($key,&DWP(24,$key));
+
+ &movdqa ("xmm3","xmm0");
+ &pslldq ("xmm0",4);
+ &pxor ("xmm3","xmm0");
+ &pslldq ("xmm0",4);
+ &pxor ("xmm3","xmm0");
+ &pslldq ("xmm0",4);
+ &pxor ("xmm0","xmm3");
+
+ &pshufd ("xmm3","xmm0",0xff);
+ &pxor ("xmm3","xmm1");
+ &pslldq ("xmm1",4);
+ &pxor ("xmm3","xmm1");
+
+ &pxor ("xmm0","xmm2");
+ &pxor ("xmm2","xmm3");
+ &movdqu (&QWP(-16,$key),"xmm0");
+
+ &dec ($rounds);
+ &jnz (&label("loop_key192"));
+
+ &mov ($rounds,11);
+ &mov (&DWP(32,$key),$rounds);
+
+ &jmp (&label("good_key"));
+
&set_label("14rounds",16);
&movups ("xmm2",&QWP(16,"eax")); # remaining half of *userKey
- &mov ($rounds,13);
&lea ($key,&DWP(16,$key));
+ &cmp ("ebp",1<<28);
+ &je (&label("14rounds_alt"));
+
+ &mov ($rounds,13);
&$movekey (&QWP(-32,$key),"xmm0"); # round 0
&$movekey (&QWP(-16,$key),"xmm2"); # round 1
&aeskeygenassist("xmm1","xmm2",0x01); # round 2
&$movekey (&QWP(0,$key),"xmm0");
&mov (&DWP(16,$key),$rounds);
&xor ("eax","eax");
- &ret();
+
+ &jmp (&label("good_key"));
&set_label("key_256a",16);
&$movekey (&QWP(0,$key),"xmm2");
&xorps ("xmm2","xmm1");
&ret();
+&set_label("14rounds_alt",16);
+ &movdqa ("xmm5",&QWP(0x00,"ebx"));
+ &movdqa ("xmm4",&QWP(0x20,"ebx"));
+ &mov ($rounds,7);
+ &movdqu (&QWP(-32,$key),"xmm0");
+ &movdqa ("xmm1","xmm2");
+ &movdqu (&QWP(-16,$key),"xmm2");
+
+&set_label("loop_key256");
+ &pshufb ("xmm2","xmm5");
+ &aesenclast ("xmm2","xmm4");
+
+ &movdqa ("xmm3","xmm0");
+ &pslldq ("xmm0",4);
+ &pxor ("xmm3","xmm0");
+ &pslldq ("xmm0",4);
+ &pxor ("xmm3","xmm0");
+ &pslldq ("xmm0",4);
+ &pxor ("xmm0","xmm3");
+ &pslld ("xmm4",1);
+
+ &pxor ("xmm0","xmm2");
+ &movdqu (&QWP(0,$key),"xmm0");
+
+ &dec ($rounds);
+ &jz (&label("done_key256"));
+
+ &pshufd ("xmm2","xmm0",0xff);
+ &pxor ("xmm3","xmm3");
+ &aesenclast ("xmm2","xmm3");
+
+ &movdqa ("xmm3","xmm1");
+ &pslldq ("xmm1",4);
+ &pxor ("xmm3","xmm1");
+ &pslldq ("xmm1",4);
+ &pxor ("xmm3","xmm1");
+ &pslldq ("xmm1",4);
+ &pxor ("xmm1","xmm3");
+
+ &pxor ("xmm2","xmm1");
+ &movdqu (&QWP(16,$key),"xmm2");
+ &lea ($key,&DWP(32,$key));
+ &movdqa ("xmm1","xmm2");
+ &jmp (&label("loop_key256"));
+
+&set_label("done_key256");
+ &mov ($rounds,13);
+ &mov (&DWP(16,$key),$rounds);
+
+&set_label("good_key");
+ &pxor ("xmm0","xmm0");
+ &pxor ("xmm1","xmm1");
+ &pxor ("xmm2","xmm2");
+ &pxor ("xmm3","xmm3");
+ &pxor ("xmm4","xmm4");
+ &pxor ("xmm5","xmm5");
+ &xor ("eax","eax");
+ &pop ("ebx");
+ &pop ("ebp");
+ &ret ();
+
&set_label("bad_pointer",4);
&mov ("eax",-1);
+ &pop ("ebx");
+ &pop ("ebp");
&ret ();
&set_label("bad_keybits",4);
+ &pxor ("xmm0","xmm0");
&mov ("eax",-2);
+ &pop ("ebx");
+ &pop ("ebp");
&ret ();
&function_end_B("_aesni_set_encrypt_key");
&aesimc ("xmm0","xmm0");
&$movekey (&QWP(0,$key),"xmm0");
+ &pxor ("xmm0","xmm0");
+ &pxor ("xmm1","xmm1");
&xor ("eax","eax"); # return success
&set_label("dec_key_ret");
&ret ();
&function_end_B("${PREFIX}_set_decrypt_key");
+
+&set_label("key_const",64);
+&data_word(0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d);
+&data_word(0x04070605,0x04070605,0x04070605,0x04070605);
+&data_word(1,1,1,1);
+&data_word(0x1b,0x1b,0x1b,0x1b);
&asciz("AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>");
&asm_finish();