From: Andy Polyakov Date: Sun, 3 Jul 2016 14:53:14 +0000 (+0200) Subject: aes/asm/aesfx-sparcv9.pl: switch to fshiftorx to improve single-block X-Git-Tag: OpenSSL_1_1_0-pre6~211 X-Git-Url: https://git.openssl.org/?p=openssl.git;a=commitdiff_plain;h=9515accaf973ad314e03bdc048b13ec3509cb2e9 aes/asm/aesfx-sparcv9.pl: switch to fshiftorx to improve single-block and short-input performance. [Fix bug in misaligned output handling.] Reviewed-by: Richard Levitte --- diff --git a/crypto/aes/asm/aesfx-sparcv9.pl b/crypto/aes/asm/aesfx-sparcv9.pl old mode 100755 new mode 100644 index 5d8d9b69ff..04b3cf7116 --- a/crypto/aes/asm/aesfx-sparcv9.pl +++ b/crypto/aes/asm/aesfx-sparcv9.pl @@ -26,6 +26,12 @@ # yet. CBC encrypt on the other hand is as good as it can possibly # get processing one byte in 4.1 cycles with 128-bit key on SPARC64 X. # This is ~6x faster than pure software implementation... +# +# July 2016 +# +# Switch from faligndata to fshiftorx, which allows to omit alignaddr +# instructions and improve single-block and short-input performance +# with misaligned data. $output = pop; open STDOUT,">$output"; @@ -45,18 +51,23 @@ $code.=<<___; aes_fx_encrypt: and $inp, 7, $tmp ! is input aligned? andn $inp, 7, $inp - ld [$key + 240], $rounds ldd [$key + 0], %f6 ! round[0] ldd [$key + 8], %f8 + mov %o7, %g1 + ld [$key + 240], $rounds +1: call .+8 + add %o7, .Linp_align-1b, %o7 + + sll $tmp, 3, $tmp ldd [$inp + 0], %f0 ! load input brz,pt $tmp, .Lenc_inp_aligned ldd [$inp + 8], %f2 + ldd [%o7 + $tmp], %f14 ! shift left params ldd [$inp + 16], %f4 - alignaddr $inp, $tmp, %g0 - faligndata %f0, %f2, %f0 - faligndata %f2, %f4, %f2 + fshiftorx %f0, %f2, %f14, %f0 + fshiftorx %f2, %f4, %f14, %f2 .Lenc_inp_aligned: ldd [$key + 16], %f10 ! round[1] @@ -87,17 +98,23 @@ aes_fx_encrypt: sub $rounds, 2, $rounds andcc $out, 7, $tmp ! is output aligned? + andn $out, 7, $out mov 0xff, $mask + srl $mask, $tmp, $mask + add %o7, 64, %o7 + sll $tmp, 3, $tmp fmovd %f0, %f4 faesencx %f2, %f10, %f0 faesencx %f4, %f12, %f2 + ldd [%o7 + $tmp], %f14 ! shift right params + fmovd %f0, %f4 faesenclx %f2, %f6, %f0 faesenclx %f4, %f8, %f2 - bnz,a,pn %icc, .Lenc_out_unaligned - srl $mask, $tmp, $mask + bnz,pn %icc, .Lenc_out_unaligned + mov %g1, %o7 std %f0, [$out + 0] retl @@ -105,16 +122,15 @@ aes_fx_encrypt: .align 16 .Lenc_out_unaligned: - alignaddrl $out, %g0, $out - faligndata %f0, %f0, %f4 - faligndata %f0, %f2, %f6 - faligndata %f2, %f2, %f8 + add $out, 16, $inp + orn %g0, $mask, $tmp + fshiftorx %f0, %f0, %f14, %f4 + fshiftorx %f0, %f2, %f14, %f6 + fshiftorx %f2, %f2, %f14, %f8 stda %f4, [$out + $mask]0xc0 ! partial store std %f6, [$out + 8] - add $out, 16, $out - orn %g0, $mask, $mask - stda %f8, [$out + $mask]0xc0 ! partial store + stda %f8, [$inp + $tmp]0xc0 ! partial store retl nop .type aes_fx_encrypt,#function @@ -125,18 +141,23 @@ aes_fx_encrypt: aes_fx_decrypt: and $inp, 7, $tmp ! is input aligned? andn $inp, 7, $inp - ld [$key + 240], $rounds ldd [$key + 0], %f6 ! round[0] ldd [$key + 8], %f8 + mov %o7, %g1 + ld [$key + 240], $rounds + +1: call .+8 + add %o7, .Linp_align-1b, %o7 + sll $tmp, 3, $tmp ldd [$inp + 0], %f0 ! load input brz,pt $tmp, .Ldec_inp_aligned ldd [$inp + 8], %f2 + ldd [%o7 + $tmp], %f14 ! shift left params ldd [$inp + 16], %f4 - alignaddr $inp, $tmp, $inp - faligndata %f0, %f2, %f0 - faligndata %f2, %f4, %f2 + fshiftorx %f0, %f2, %f14, %f0 + fshiftorx %f2, %f4, %f14, %f2 .Ldec_inp_aligned: ldd [$key + 16], %f10 ! round[1] @@ -167,17 +188,23 @@ aes_fx_decrypt: sub $rounds, 2, $rounds andcc $out, 7, $tmp ! is output aligned? + andn $out, 7, $out mov 0xff, $mask + srl $mask, $tmp, $mask + add %o7, 64, %o7 + sll $tmp, 3, $tmp fmovd %f0, %f4 faesdecx %f2, %f10, %f0 faesdecx %f4, %f12, %f2 + ldd [%o7 + $tmp], %f14 ! shift right params + fmovd %f0, %f4 faesdeclx %f2, %f6, %f0 faesdeclx %f4, %f8, %f2 - bnz,a,pn %icc, .Ldec_out_unaligned - srl $mask, $tmp, $mask + bnz,pn %icc, .Ldec_out_unaligned + mov %g1, %o7 std %f0, [$out + 0] retl @@ -185,16 +212,15 @@ aes_fx_decrypt: .align 16 .Ldec_out_unaligned: - alignaddrl $out, %g0, $out - faligndata %f0, %f0, %f4 - faligndata %f0, %f2, %f6 - faligndata %f2, %f2, %f8 + add $out, 16, $inp + orn %g0, $mask, $tmp + fshiftorx %f0, %f0, %f14, %f4 + fshiftorx %f0, %f2, %f14, %f6 + fshiftorx %f2, %f2, %f14, %f8 stda %f4, [$out + $mask]0xc0 ! partial store std %f6, [$out + 8] - add $out, 16, $out - orn %g0, $mask, $mask - stda %f8, [$out + $mask]0xc0 ! partial store + stda %f8, [$inp + $tmp]0xc0 ! partial store retl nop .type aes_fx_decrypt,#function @@ -222,6 +248,14 @@ aes_fx_set_encrypt_key: .Lset_encrypt_key: and $inp, 7, $tmp andn $inp, 7, $inp + sll $tmp, 3, $tmp + mov %o7, %g1 + +1: call .+8 + add %o7, .Linp_align-1b, %o7 + + ldd [%o7 + $tmp], %f10 ! shift left params + mov %g1, %o7 cmp $bits, 192 ldd [$inp + 0], %f0 @@ -234,11 +268,10 @@ aes_fx_set_encrypt_key: ldd [$inp + 24], %f6 ldd [$inp + 32], %f8 - alignaddr $inp, $tmp, %g0 - faligndata %f0, %f2, %f0 - faligndata %f2, %f4, %f2 - faligndata %f4, %f6, %f4 - faligndata %f6, %f8, %f6 + fshiftorx %f0, %f2, %f10, %f0 + fshiftorx %f2, %f4, %f10, %f2 + fshiftorx %f4, %f6, %f10, %f4 + fshiftorx %f6, %f8, %f10, %f6 .L256aligned: mov 14, $bits @@ -281,10 +314,9 @@ $code.=<<___; nop ldd [$inp + 24], %f6 - alignaddr $inp, $tmp, %g0 - faligndata %f0, %f2, %f0 - faligndata %f2, %f4, %f2 - faligndata %f4, %f6, %f4 + fshiftorx %f0, %f2, %f10, %f0 + fshiftorx %f2, %f4, %f10, %f2 + fshiftorx %f4, %f6, %f10, %f4 .L192aligned: mov 12, $bits @@ -326,9 +358,8 @@ $code.=<<___; nop ldd [$inp + 16], %f4 - alignaddr $inp, $tmp, %g0 - faligndata %f0, %f2, %f0 - faligndata %f2, %f4, %f2 + fshiftorx %f0, %f2, %f10, %f0 + fshiftorx %f2, %f4, %f10, %f2 .L128aligned: mov 10, $bits @@ -358,7 +389,7 @@ ___ { my ($inp,$out,$len,$key,$ivp,$dir) = map("%i$_",(0..5)); my ($rounds,$inner,$end,$inc,$ialign,$oalign,$mask) = map("%l$_",(0..7)); -my ($out0,$out1,$iv0,$iv1,$r0hi,$r0lo,$rlhi,$rllo,$in0,$in1,$intail,$outhead) +my ($iv0,$iv1,$r0hi,$r0lo,$rlhi,$rllo,$in0,$in1,$intail,$outhead,$fshift) = map("%f$_",grep { !($_ & 1) } (16 .. 62)); my ($ileft,$iright) = ($ialign,$oalign); @@ -368,14 +399,20 @@ $code.=<<___; aes_fx_cbc_encrypt: save %sp, -STACK_FRAME-16, %sp srln $len, 4, $len - brz,pn $len, .Lcbc_no_data and $inp, 7, $ialign - andn $inp, 7, $inp + brz,pn $len, .Lcbc_no_data + sll $ialign, 3, $ileft + +1: call .+8 + add %o7, .Linp_align-1b, %o7 + ld [$key + 240], $rounds and $out, 7, $oalign ld [$ivp + 0], %f0 ! load ivec + andn $out, 7, $out ld [$ivp + 4], %f1 + sll $oalign, 3, $mask ld [$ivp + 8], %f2 ld [$ivp + 12], %f3 @@ -394,6 +431,8 @@ aes_fx_cbc_encrypt: ldd [$key + 16], %f10 ! round[1] ldd [$key + 24], %f12 + ldd [%o7 + $ileft], $fshift ! shift left params + add %o7, 64, %o7 ldd [$inp - 16], $in0 ! load input ldd [$inp - 8], $in1 ldda [$inp]0x82, $intail ! non-faulting load @@ -402,11 +441,9 @@ aes_fx_cbc_encrypt: fxor $r0hi, %f0, %f0 ! ivec^=round[0] fxor $r0lo, %f2, %f2 - alignaddr $inp, $ialign, %g0 - faligndata $in0, $in1, $in0 - faligndata $in1, $intail, $in1 - fxor $r0hi, $rlhi, $rlhi ! round[last]^=round[0] - fxor $r0lo, $rllo, $rllo + fshiftorx $in0, $in1, $fshift, $in0 + fshiftorx $in1, $intail, $fshift, $in1 + nop .Loop_cbc_enc: fxor $in0, %f0, %f0 ! inp^ivec^round[0] @@ -439,44 +476,46 @@ aes_fx_cbc_encrypt: ldd [$end + 16], %f10 ! round[last-1] ldd [$end + 24], %f12 - fmovd %f0, %f4 - faesencx %f2, %f6, %f0 - faesencx %f4, %f8, %f2 - movrz $len, 0, $inc fmovd $intail, $in0 ldd [$inp - 8], $in1 ! load next input block ldda [$inp]0x82, $intail ! non-faulting load add $inp, $inc, $inp ! inp+=16 + fmovd %f0, %f4 + faesencx %f2, %f6, %f0 + faesencx %f4, %f8, %f2 + + fshiftorx $in0, $in1, $fshift, $in0 + fshiftorx $in1, $intail, $fshift, $in1 + fmovd %f0, %f4 faesencx %f2, %f10, %f0 faesencx %f4, %f12, %f2 ldd [$key + 16], %f10 ! round[1] ldd [$key + 24], %f12 - faligndata $in0, $in1, $in0 - faligndata $in1, $intail, $in1 + fxor $r0hi, $in0, $in0 ! inp^=round[0] + fxor $r0lo, $in1, $in1 fmovd %f0, %f4 - faesenclx %f2, $rlhi, %f0 ! result is out^round[0] + faesenclx %f2, $rlhi, %f0 faesenclx %f4, $rllo, %f2 - fxor %f0, $r0hi, $out0 ! out^round[0]^round[0] brnz,pn $oalign, .Lcbc_enc_unaligned_out - fxor %f2, $r0lo, $out1 + nop - std $out0, [$out + 0] - std $out1, [$out + 8] + std %f0, [$out + 0] + std %f2, [$out + 8] add $out, 16, $out brnz,a $len, .Loop_cbc_enc sub $len, 1, $len - st $out0, [$ivp + 0] ! output ivec - st $out0#lo, [$ivp + 4] - st $out1, [$ivp + 8] - st $out1#lo, [$ivp + 12] + st %f0, [$ivp + 0] ! output ivec + st %f1, [$ivp + 4] + st %f2, [$ivp + 8] + st %f3, [$ivp + 12] .Lcbc_no_data: ret @@ -484,22 +523,26 @@ aes_fx_cbc_encrypt: .align 32 .Lcbc_enc_unaligned_out: - alignaddrl $out, %g0, $out + ldd [%o7 + $mask], $fshift ! shift right params mov 0xff, $mask - sll $ialign, 3, $ileft srl $mask, $oalign, $mask sub %g0, $ileft, $iright - faligndata $out0, $out0, %f6 - faligndata $out0, $out1, %f8 + fshiftorx %f0, %f0, $fshift, %f6 + fshiftorx %f0, %f2, $fshift, %f8 stda %f6, [$out + $mask]0xc0 ! partial store + orn %g0, $mask, $mask std %f8, [$out + 8] add $out, 16, $out brz $len, .Lcbc_enc_unaligned_out_done - orn %g0, $mask, $mask + sub $len, 1, $len + b .Loop_cbc_enc_unaligned_out + nop +.align 32 .Loop_cbc_enc_unaligned_out: + fmovd %f2, $outhead fxor $in0, %f0, %f0 ! inp^ivec^round[0] fxor $in1, %f2, %f2 ldd [$key + 32], %f6 ! round[2] @@ -513,7 +556,7 @@ aes_fx_cbc_encrypt: ldx [$inp - 16], %o0 ldx [$inp - 8], %o1 - brz $ialign, .Lcbc_enc_aligned_inp + brz $ileft, .Lcbc_enc_aligned_inp movrz $len, 0, $inc ldx [$inp], %o2 @@ -536,6 +579,7 @@ aes_fx_cbc_encrypt: stx %o0, [%sp + LOCALS + 0] stx %o1, [%sp + LOCALS + 8] add $inp, $inc, $inp ! inp+=16 + nop .Lcbc_enc_unaligned: fmovd %f0, %f4 @@ -563,6 +607,7 @@ aes_fx_cbc_encrypt: fmovd %f0, %f4 faesencx %f2, %f6, %f0 faesencx %f4, %f8, %f2 + ldd [%sp + LOCALS + 0], $in0 ldd [%sp + LOCALS + 8], $in1 @@ -572,16 +617,15 @@ aes_fx_cbc_encrypt: ldd [$key + 16], %f10 ! round[1] ldd [$key + 24], %f12 + fxor $r0hi, $in0, $in0 ! inp^=round[0] + fxor $r0lo, $in1, $in1 + fmovd %f0, %f4 - faesenclx %f2, $rlhi, %f0 ! result is out^round[0] + faesenclx %f2, $rlhi, %f0 faesenclx %f4, $rllo, %f2 - fmovd $out1, $outhead - fxor %f0, $r0hi, $out0 ! out^round[0]^round[0] - fxor %f2, $r0lo, $out1 - - faligndata $outhead, $out0, %f6 - faligndata $out0, $out1, %f8 + fshiftorx $outhead, %f0, $fshift, %f6 + fshiftorx %f0, %f2, $fshift, %f8 std %f6, [$out + 0] std %f8, [$out + 8] add $out, 16, $out @@ -590,22 +634,21 @@ aes_fx_cbc_encrypt: sub $len, 1, $len .Lcbc_enc_unaligned_out_done: - faligndata $out1, $out1, %f8 + fshiftorx %f2, %f2, $fshift, %f8 stda %f8, [$out + $mask]0xc0 ! partial store - st $out0, [$ivp + 0] ! output ivec - st $out0#lo, [$ivp + 4] - st $out1, [$ivp + 8] - st $out1#lo, [$ivp + 12] + st %f0, [$ivp + 0] ! output ivec + st %f1, [$ivp + 4] + st %f2, [$ivp + 8] + st %f3, [$ivp + 12] ret restore .align 32 .Lcbc_decrypt: - alignaddr $inp, $ialign, %g0 - faligndata $in0, $in1, $in0 - faligndata $in1, $intail, $in1 + fshiftorx $in0, $in1, $fshift, $in0 + fshiftorx $in1, $intail, $fshift, $in1 fmovd %f0, $iv0 fmovd %f2, $iv1 @@ -660,8 +703,8 @@ aes_fx_cbc_encrypt: ldd [$key + 16], %f10 ! round[1] ldd [$key + 24], %f12 - faligndata $in0, $in1, $in0 - faligndata $in1, $intail, $in1 + fshiftorx $in0, $in1, $fshift, $in0 + fshiftorx $in1, $intail, $fshift, $in1 fmovd %f0, %f4 faesdeclx %f2, %f6, %f0 @@ -687,21 +730,24 @@ aes_fx_cbc_encrypt: .align 32 .Lcbc_dec_unaligned_out: - alignaddrl $out, %g0, $out + ldd [%o7 + $mask], $fshift ! shift right params mov 0xff, $mask - sll $ialign, 3, $ileft srl $mask, $oalign, $mask sub %g0, $ileft, $iright - faligndata %f0, %f0, $out0 - faligndata %f0, %f2, $out1 + fshiftorx %f0, %f0, $fshift, %f6 + fshiftorx %f0, %f2, $fshift, %f8 - stda $out0, [$out + $mask]0xc0 ! partial store - std $out1, [$out + 8] + stda %f6, [$out + $mask]0xc0 ! partial store + orn %g0, $mask, $mask + std %f8, [$out + 8] add $out, 16, $out brz $len, .Lcbc_dec_unaligned_out_done - orn %g0, $mask, $mask + sub $len, 1, $len + b .Loop_cbc_dec_unaligned_out + nop +.align 32 .Loop_cbc_dec_unaligned_out: fmovd %f2, $outhead fxor $in0, $r0hi, %f0 ! inp^round[0] @@ -717,7 +763,7 @@ aes_fx_cbc_encrypt: ldx [$inp - 16], %o0 ldx [$inp - 8], %o1 - brz $ialign, .Lcbc_dec_aligned_inp + brz $ileft, .Lcbc_dec_aligned_inp movrz $len, 0, $inc ldx [$inp], %o2 @@ -740,6 +786,7 @@ aes_fx_cbc_encrypt: stx %o0, [%sp + LOCALS + 0] stx %o1, [%sp + LOCALS + 8] add $inp, $inc, $inp ! inp+=16 + nop .Lcbc_dec_unaligned: fmovd %f0, %f4 @@ -767,10 +814,13 @@ aes_fx_cbc_encrypt: fmovd %f0, %f4 faesdecx %f2, %f6, %f0 faesdecx %f4, %f8, %f2 + fxor $iv0, $rlhi, %f6 ! ivec^round[last] fxor $iv1, $rllo, %f8 fmovd $in0, $iv0 fmovd $in1, $iv1 + ldd [%sp + LOCALS + 0], $in0 + ldd [%sp + LOCALS + 8], $in1 fmovd %f0, %f4 faesdecx %f2, %f10, %f0 @@ -781,20 +831,18 @@ aes_fx_cbc_encrypt: fmovd %f0, %f4 faesdeclx %f2, %f6, %f0 faesdeclx %f4, %f8, %f2 - ldd [%sp + LOCALS + 0], $in0 - ldd [%sp + LOCALS + 8], $in1 - faligndata $outhead, %f0, $out0 - faligndata %f0, %f2, $out1 - std $out0, [$out + 0] - std $out1, [$out + 8] + fshiftorx $outhead, %f0, $fshift, %f6 + fshiftorx %f0, %f2, $fshift, %f8 + std %f6, [$out + 0] + std %f8, [$out + 8] add $out, 16, $out brnz,a $len, .Loop_cbc_dec_unaligned_out sub $len, 1, $len .Lcbc_dec_unaligned_out_done: - faligndata %f2, %f2, %f8 + fshiftorx %f2, %f2, $fshift, %f8 stda %f8, [$out + $mask]0xc0 ! partial store st $iv0, [$ivp + 0] ! output ivec @@ -811,7 +859,7 @@ ___ { my ($inp,$out,$len,$key,$ivp) = map("%i$_",(0..5)); my ($rounds,$inner,$end,$inc,$ialign,$oalign,$mask) = map("%l$_",(0..7)); -my ($out0,$out1,$ctr0,$ctr1,$r0hi,$r0lo,$rlhi,$rllo,$in0,$in1,$intail,$outhead) +my ($ctr0,$ctr1,$r0hi,$r0lo,$rlhi,$rllo,$in0,$in1,$intail,$outhead,$fshift) = map("%f$_",grep { !($_ & 1) } (16 .. 62)); my ($ileft,$iright) = ($ialign, $oalign); my $one = "%f14"; @@ -822,22 +870,23 @@ $code.=<<___; aes_fx_ctr32_encrypt_blocks: save %sp, -STACK_FRAME-16, %sp srln $len, 0, $len - brz,pn $len, .Lctr32_no_data - nop - and $inp, 7, $ialign andn $inp, 7, $inp + brz,pn $len, .Lctr32_no_data + sll $ialign, 3, $ileft .Lpic: call .+8 - add %o7, .Lone - .Lpic, %o0 + add %o7, .Linp_align - .Lpic, %o7 ld [$key + 240], $rounds and $out, 7, $oalign ld [$ivp + 0], $ctr0 ! load counter + andn $out, 7, $out ld [$ivp + 4], $ctr0#lo + sll $oalign, 3, $mask ld [$ivp + 8], $ctr1 ld [$ivp + 12], $ctr1#lo - ldd [%o0], $one + ldd [%o7 + 128], $one sll $rounds, 4, $rounds add $rounds, $key, $end @@ -854,14 +903,15 @@ aes_fx_ctr32_encrypt_blocks: ldd [$end + 0], $rlhi ! round[last] ldd [$end + 8], $rllo + ldd [%o7 + $ileft], $fshift ! shiftleft params + add %o7, 64, %o7 ldd [$inp - 16], $in0 ! load input ldd [$inp - 8], $in1 ldda [$inp]0x82, $intail ! non-faulting load add $inp, $inc, $inp ! inp+=16 - alignaddr $inp, $ialign, %g0 - faligndata $in0, $in1, $in0 - faligndata $in1, $intail, $in1 + fshiftorx $in0, $in1, $fshift, $in0 + fshiftorx $in1, $intail, $fshift, $in1 .Loop_ctr32: fxor $ctr0, $r0hi, %f0 ! counter^round[0] @@ -912,8 +962,8 @@ aes_fx_ctr32_encrypt_blocks: ldd [$key + 16], %f10 ! round[1] ldd [$key + 24], %f12 - faligndata $in0, $in1, $in0 - faligndata $in1, $intail, $in1 + fshiftorx $in0, $in1, $fshift, $in0 + fshiftorx $in1, $intail, $fshift, $in1 fpadd32 $ctr1, $one, $ctr1 ! increment counter fmovd %f0, %f4 @@ -936,21 +986,24 @@ aes_fx_ctr32_encrypt_blocks: .align 32 .Lctr32_unaligned_out: - alignaddrl $out, %g0, $out + ldd [%o7 + $mask], $fshift ! shift right params mov 0xff, $mask - sll $ialign, 3, $ileft srl $mask, $oalign, $mask sub %g0, $ileft, $iright - faligndata %f0, %f0, $out0 - faligndata %f0, %f2, $out1 + fshiftorx %f0, %f0, $fshift, %f6 + fshiftorx %f0, %f2, $fshift, %f8 - stda $out0, [$out + $mask]0xc0 ! partial store - std $out1, [$out + 8] + stda %f6, [$out + $mask]0xc0 ! partial store + orn %g0, $mask, $mask + std %f8, [$out + 8] add $out, 16, $out brz $len, .Lctr32_unaligned_out_done - orn %g0, $mask, $mask + sub $len, 1, $len + b .Loop_ctr32_unaligned_out + nop +.align 32 .Loop_ctr32_unaligned_out: fmovd %f2, $outhead fxor $ctr0, $r0hi, %f0 ! counter^round[0] @@ -966,7 +1019,7 @@ aes_fx_ctr32_encrypt_blocks: ldx [$inp - 16], %o0 ldx [$inp - 8], %o1 - brz $ialign, .Lctr32_aligned_inp + brz $ileft, .Lctr32_aligned_inp movrz $len, 0, $inc ldx [$inp], %o2 @@ -989,6 +1042,7 @@ aes_fx_ctr32_encrypt_blocks: stx %o0, [%sp + LOCALS + 0] stx %o1, [%sp + LOCALS + 8] add $inp, $inc, $inp ! inp+=16 + nop .Lctr32_enc_unaligned: fmovd %f0, %f4 @@ -1032,24 +1086,43 @@ aes_fx_ctr32_encrypt_blocks: faesenclx %f2, %f6, %f0 faesenclx %f4, %f8, %f2 - faligndata $outhead, %f0, $out0 - faligndata %f0, %f2, $out1 - std $out0, [$out + 0] - std $out1, [$out + 8] + fshiftorx $outhead, %f0, $fshift, %f6 + fshiftorx %f0, %f2, $fshift, %f8 + std %f6, [$out + 0] + std %f8, [$out + 8] add $out, 16, $out brnz,a $len, .Loop_ctr32_unaligned_out sub $len, 1, $len .Lctr32_unaligned_out_done: - faligndata %f2, %f2, %f8 + fshiftorx %f2, %f2, $fshift, %f8 stda %f8, [$out + $mask]0xc0 ! partial store ret restore .type aes_fx_ctr32_encrypt_blocks,#function .size aes_fx_ctr32_encrypt_blocks,.-aes_fx_ctr32_encrypt_blocks + .align 32 +.Linp_align: ! fshiftorx parameters for left shift toward %rs1 + .byte 0, 0, 64, 0, 0, 64, 0, -64 + .byte 0, 0, 56, 8, 0, 56, 8, -56 + .byte 0, 0, 48, 16, 0, 48, 16, -48 + .byte 0, 0, 40, 24, 0, 40, 24, -40 + .byte 0, 0, 32, 32, 0, 32, 32, -32 + .byte 0, 0, 24, 40, 0, 24, 40, -24 + .byte 0, 0, 16, 48, 0, 16, 48, -16 + .byte 0, 0, 8, 56, 0, 8, 56, -8 +.Lout_align: ! fshiftorx parameters for right shift toward %rs2 + .byte 0, 0, 0, 64, 0, 0, 64, 0 + .byte 0, 0, 8, 56, 0, 8, 56, -8 + .byte 0, 0, 16, 48, 0, 16, 48, -16 + .byte 0, 0, 24, 40, 0, 24, 40, -24 + .byte 0, 0, 32, 32, 0, 32, 32, -32 + .byte 0, 0, 40, 24, 0, 40, 24, -40 + .byte 0, 0, 48, 16, 0, 48, 16, -48 + .byte 0, 0, 56, 8, 0, 56, 8, -56 .Lone: .word 0, 1 .asciz "AES for Fujitsu SPARC64 X, CRYPTOGAMS by " @@ -1148,13 +1221,42 @@ my %aesopf = ( "faesencx" => 0x90, } } +sub unfx3src { +my ($mnemonic,$rs1,$rs2,$rs3,$rd)=@_; +my ($ref,$opf); +my %aesopf = ( "fshiftorx" => 0x0b ); + + $ref = "$mnemonic\t$rs1,$rs2,$rs3,$rd"; + + if (defined($opf=$aesopf{$mnemonic})) { + foreach ($rs1,$rs2,$rs3,$rd) { + return $ref if (!/%f([0-9]{1,2})/); + $_=$1; + if ($1>=32) { + return $ref if ($1&1); + # re-encode for upper double register addressing + $_=($1|$1>>5)&31; + } + } + + return sprintf ".word\t0x%08x !%s", + 2<<30|$rd<<25|0x37<<19|$rs1<<14|$rs3<<9|$opf<<5|$rs2, + $ref; + } else { + return $ref; + } +} + foreach (split("\n",$code)) { s/\`([^\`]*)\`/eval $1/ge; s/%f([0-9]+)#lo/sprintf "%%f%d",$1+1/ge; s/\b(faes[^x]{3,4}x)\s+(%f[0-9]{1,2}),\s*([%fx0-9]+),\s*(%f[0-9]{1,2})/ - &unfx($1,$2,$3,$4,$5) + &unfx($1,$2,$3,$4) + /ge or + s/\b([f][^\s]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/ + &unfx3src($1,$2,$3,$4,$5) /ge or s/\b([fb][^\s]*)\s+(%f[0-9]{1,2}),\s*(%f[0-9]{1,2}),\s*(%f[0-9]{1,2})/ &unvis($1,$2,$3,$4)