+ faligndata $outhead, $out0, %f6
+ faligndata $out0, $out1, %f8
+ std %f6, [$out + 0]
+ std %f8, [$out + 8]
+ add $out, 16, $out
+
+ brnz,a $len, .Loop_cbc_enc_unaligned_out
+ sub $len, 16, $len
+
+.Lcbc_enc_unaligned_out_done:
+ faligndata $out1, $out1, %f8
+ stda %f8, [$out + $mask]0xc0 ! partial store
+
+ st $out0, [$ivp + 0] ! output ivec
+ st $out0#lo, [$ivp + 4]
+ st $out1, [$ivp + 8]
+ st $out1#lo, [$ivp + 12]
+
+ ret
+ restore
+
+.align 32
+.Lcbc_decrypt:
+ alignaddr $inp, $ialign, %g0
+ faligndata $in0, $in1, $in0
+ faligndata $in1, $intail, $in1
+ fmovd %f0, $iv0
+ fmovd %f2, $iv1
+
+.Loop_cbc_dec:
+ fxor $in0, $r0hi, %f0 ! inp^round[0]
+ fxor $in1, $r0lo, %f2
+ ldd [$key + 32], %f6 ! round[2]
+ ldd [$key + 40], %f8
+ add $key, 32, $end
+ sub $rounds, 16*6, $inner
+
+.Lcbc_dec:
+ fmovd %f0, %f4
+ faesdecx %f2, %f10, %f0
+ faesdecx %f4, %f12, %f2
+ ldd [$end + 16], %f10
+ ldd [$end + 24], %f12
+ add $end, 32, $end
+
+ fmovd %f0, %f4
+ faesdecx %f2, %f6, %f0
+ faesdecx %f4, %f8, %f2
+ ldd [$end + 0], %f6
+ ldd [$end + 8], %f8
+
+ brnz,a $inner, .Lcbc_dec
+ sub $inner, 16*2, $inner
+
+ fmovd %f0, %f4
+ faesdecx %f2, %f10, %f0
+ faesdecx %f4, %f12, %f2
+ ldd [$end + 16], %f10 ! round[last-1]
+ ldd [$end + 24], %f12
+
+ fmovd %f0, %f4
+ faesdecx %f2, %f6, %f0
+ faesdecx %f4, %f8, %f2
+ fxor $iv0, $rlhi, %f6 ! ivec^round[last]
+ fxor $iv1, $rllo, %f8
+ fmovd $in0, $iv0
+ fmovd $in1, $iv1
+
+ movrz $len, 0, $inc
+ fmovd $intail, $in0
+ ldd [$inp - 8], $in1 ! load next input block
+ ldda [$inp]0x82, $intail ! non-faulting load
+ add $inp, $inc, $inp ! inp+=16
+
+ fmovd %f0, %f4
+ faesdecx %f2, %f10, %f0
+ faesdecx %f4, %f12, %f2
+ ldd [$key + 16], %f10 ! round[1]
+ ldd [$key + 24], %f12
+
+ faligndata $in0, $in1, $in0
+ faligndata $in1, $intail, $in1
+
+ fmovd %f0, %f4
+ faesdeclx %f2, %f6, %f0
+ faesdeclx %f4, %f8, %f2
+
+ brnz,pn $oalign, .Lcbc_dec_unaligned_out
+ nop
+
+ std %f0, [$out + 0]
+ std %f2, [$out + 8]
+ add $out, 16, $out
+
+ brnz,a $len, .Loop_cbc_dec
+ sub $len, 16, $len
+
+ st $iv0, [$ivp + 0] ! output ivec
+ st $iv0#lo, [$ivp + 4]
+ st $iv1, [$ivp + 8]
+ st $iv1#lo, [$ivp + 12]
+
+ ret
+ restore
+
+.align 32
+.Lcbc_dec_unaligned_out:
+ alignaddrl $out, %g0, $out
+ mov 0xff, $mask
+ sll $ialign, 3, $ileft
+ srl $mask, $oalign, $mask
+ sub %g0, $ileft, $iright
+
+ faligndata %f0, %f0, $out0
+ faligndata %f0, %f2, $out1
+
+ stda $out0, [$out + $mask]0xc0 ! partial store
+ std $out1, [$out + 8]
+ add $out, 16, $out
+ brz $len, .Lcbc_dec_unaligned_out_done
+ orn %g0, $mask, $mask
+
+.Loop_cbc_dec_unaligned_out:
+ fmovd %f2, $outhead
+ fxor $in0, $r0hi, %f0 ! inp^round[0]
+ fxor $in1, $r0lo, %f2
+ ldd [$key + 32], %f6 ! round[2]
+ ldd [$key + 40], %f8
+
+ fmovd %f0, %f4
+ faesdecx %f2, %f10, %f0
+ faesdecx %f4, %f12, %f2
+ ldd [$key + 48], %f10 ! round[3]
+ ldd [$key + 56], %f12
+
+ ldx [$inp - 16], %o0
+ ldx [$inp - 8], %o1
+ brz $ialign, .Lcbc_dec_aligned_inp
+ movrz $len, 0, $inc
+
+ ldx [$inp], %o2
+ sllx %o0, $ileft, %o0
+ srlx %o1, $iright, %g1
+ sllx %o1, $ileft, %o1
+ or %g1, %o0, %o0
+ srlx %o2, $iright, %o2
+ or %o2, %o1, %o1
+
+.Lcbc_dec_aligned_inp:
+ fmovd %f0, %f4
+ faesdecx %f2, %f6, %f0
+ faesdecx %f4, %f8, %f2
+ ldd [$key + 64], %f6 ! round[4]
+ ldd [$key + 72], %f8
+ add $key, 64, $end
+ sub $rounds, 16*8, $inner
+
+ stx %o0, [%sp + LOCALS + 0]
+ stx %o1, [%sp + LOCALS + 8]
+ add $inp, $inc, $inp ! inp+=16
+
+.Lcbc_dec_unaligned:
+ fmovd %f0, %f4
+ faesdecx %f2, %f10, %f0
+ faesdecx %f4, %f12, %f2
+ ldd [$end + 16], %f10
+ ldd [$end + 24], %f12
+ add $end, 32, $end
+
+ fmovd %f0, %f4
+ faesdecx %f2, %f6, %f0
+ faesdecx %f4, %f8, %f2
+ ldd [$end + 0], %f6
+ ldd [$end + 8], %f8
+
+ brnz,a $inner, .Lcbc_dec_unaligned
+ sub $inner, 16*2, $inner
+
+ fmovd %f0, %f4
+ faesdecx %f2, %f10, %f0
+ faesdecx %f4, %f12, %f2
+ ldd [$end + 16], %f10 ! round[last-1]
+ ldd [$end + 24], %f12
+
+ fmovd %f0, %f4
+ faesdecx %f2, %f6, %f0
+ faesdecx %f4, %f8, %f2
+ fxor $iv0, $rlhi, %f6 ! ivec^round[last]
+ fxor $iv1, $rllo, %f8
+ fmovd $in0, $iv0
+ fmovd $in1, $iv1
+
+ fmovd %f0, %f4
+ faesdecx %f2, %f10, %f0
+ faesdecx %f4, %f12, %f2
+ ldd [$key + 16], %f10 ! round[1]
+ ldd [$key + 24], %f12
+
+ fmovd %f0, %f4
+ faesdeclx %f2, %f6, %f0
+ faesdeclx %f4, %f8, %f2
+ ldd [%sp + LOCALS + 0], $in0
+ ldd [%sp + LOCALS + 8], $in1
+
+ faligndata $outhead, %f0, $out0
+ faligndata %f0, %f2, $out1
+ std $out0, [$out + 0]
+ std $out1, [$out + 8]
+ add $out, 16, $out
+
+ brnz,a $len, .Loop_cbc_dec_unaligned_out
+ sub $len, 16, $len
+
+.Lcbc_dec_unaligned_out_done:
+ faligndata %f2, %f2, %f8
+ stda %f8, [$out + $mask]0xc0 ! partial store
+
+ st $iv0, [$ivp + 0] ! output ivec
+ st $iv0#lo, [$ivp + 4]
+ st $iv1, [$ivp + 8]
+ st $iv1#lo, [$ivp + 12]
+
+ ret
+ restore
+.type aes_fx_cbc_encrypt,#function
+.size aes_fx_cbc_encrypt,.-aes_fx_cbc_encrypt
+___
+}
+{
+my ($inp,$out,$len,$key,$ivp) = map("%i$_",(0..5));
+my ($rounds,$inner,$end,$inc,$ialign,$oalign,$mask) = map("%l$_",(0..7));
+my ($out0,$out1,$ctr0,$ctr1,$r0hi,$r0lo,$rlhi,$rllo,$in0,$in1,$intail,$outhead)
+ = map("%f$_",grep { !($_ & 1) } (16 .. 62));
+my ($ileft,$iright) = ($ialign, $oalign);
+my $one = "%f14";
+
+$code.=<<___;
+.globl aes_fx_ctr32_encrypt_blocks
+.align 32
+aes_fx_ctr32_encrypt_blocks:
+ save %sp, -STACK_FRAME-16, %sp
+ and $inp, 7, $ialign
+ brz,pn $len, .Lctr32_no_data
+ andn $inp, 7, $inp
+
+.Lpic: call .+8
+ add %o7, .Lone - .Lpic, %o0
+
+ ld [$key + 240], $rounds
+ and $out, 7, $oalign
+ ld [$ivp + 0], $ctr0 ! load counter
+ ld [$ivp + 4], $ctr0#lo
+ ld [$ivp + 8], $ctr1
+ ld [$ivp + 12], $ctr1#lo
+ ldd [%o0], $one
+
+ sll $rounds, 4, $rounds
+ add $rounds, $key, $end
+ ldd [$key + 0], $r0hi ! round[0]
+ ldd [$key + 8], $r0lo
+
+ add $inp, 16, $inp
+ sub $len, 1, $len
+ ldd [$key + 16], %f10 ! round[1]
+ ldd [$key + 24], %f12
+
+ mov 16, $inc
+ movrz $len, 0, $inc
+ ldd [$end + 0], $rlhi ! round[last]
+ ldd [$end + 8], $rllo
+
+ ldd [$inp - 16], $in0 ! load input
+ ldd [$inp - 8], $in1
+ ldda [$inp]0x82, $intail ! non-faulting load
+ add $inp, $inc, $inp ! inp+=16
+
+ alignaddr $inp, $ialign, %g0
+ faligndata $in0, $in1, $in0
+ faligndata $in1, $intail, $in1
+
+.Loop_ctr32:
+ fxor $ctr0, $r0hi, %f0 ! counter^round[0]
+ fxor $ctr1, $r0lo, %f2
+ ldd [$key + 32], %f6 ! round[2]
+ ldd [$key + 40], %f8
+ add $key, 32, $end
+ sub $rounds, 16*6, $inner
+
+.Lctr32_enc:
+ fmovd %f0, %f4
+ faesencx %f2, %f10, %f0
+ faesencx %f4, %f12, %f2
+ ldd [$end + 16], %f10
+ ldd [$end + 24], %f12
+ add $end, 32, $end
+
+ fmovd %f0, %f4
+ faesencx %f2, %f6, %f0
+ faesencx %f4, %f8, %f2
+ ldd [$end + 0], %f6
+ ldd [$end + 8], %f8
+
+ brnz,a $inner, .Lctr32_enc
+ sub $inner, 16*2, $inner
+
+ fmovd %f0, %f4
+ faesencx %f2, %f10, %f0
+ faesencx %f4, %f12, %f2
+ ldd [$end + 16], %f10 ! round[last-1]
+ ldd [$end + 24], %f12
+
+ fmovd %f0, %f4
+ faesencx %f2, %f6, %f0
+ faesencx %f4, %f8, %f2
+ fxor $in0, $rlhi, %f6 ! inp^round[last]
+ fxor $in1, $rllo, %f8
+
+ movrz $len, 0, $inc
+ fmovd $intail, $in0
+ ldd [$inp - 8], $in1 ! load next input block
+ ldda [$inp]0x82, $intail ! non-faulting load
+ add $inp, $inc, $inp ! inp+=16
+
+ fmovd %f0, %f4
+ faesencx %f2, %f10, %f0
+ faesencx %f4, %f12, %f2
+ ldd [$key + 16], %f10 ! round[1]
+ ldd [$key + 24], %f12
+
+ faligndata $in0, $in1, $in0
+ faligndata $in1, $intail, $in1
+ fpadd32 $ctr1, $one, $ctr1 ! increment counter
+
+ fmovd %f0, %f4
+ faesenclx %f2, %f6, %f0
+ faesenclx %f4, %f8, %f2
+
+ brnz,pn $oalign, .Lctr32_unaligned_out
+ nop
+
+ std %f0, [$out + 0]
+ std %f2, [$out + 8]
+ add $out, 16, $out
+
+ brnz,a $len, .Loop_ctr32
+ sub $len, 1, $len
+
+.Lctr32_no_data:
+ ret
+ restore
+
+.align 32
+.Lctr32_unaligned_out:
+ alignaddrl $out, %g0, $out
+ mov 0xff, $mask
+ sll $ialign, 3, $ileft
+ srl $mask, $oalign, $mask
+ sub %g0, $ileft, $iright
+
+ faligndata %f0, %f0, $out0
+ faligndata %f0, %f2, $out1
+
+ stda $out0, [$out + $mask]0xc0 ! partial store
+ std $out1, [$out + 8]
+ add $out, 16, $out
+ brz $len, .Lctr32_unaligned_out_done
+ orn %g0, $mask, $mask
+
+.Loop_ctr32_unaligned_out:
+ fmovd %f2, $outhead
+ fxor $ctr0, $r0hi, %f0 ! counter^round[0]
+ fxor $ctr1, $r0lo, %f2
+ ldd [$key + 32], %f6 ! round[2]
+ ldd [$key + 40], %f8
+
+ fmovd %f0, %f4
+ faesencx %f2, %f10, %f0
+ faesencx %f4, %f12, %f2
+ ldd [$key + 48], %f10 ! round[3]
+ ldd [$key + 56], %f12
+
+ ldx [$inp - 16], %o0
+ ldx [$inp - 8], %o1
+ brz $ialign, .Lctr32_aligned_inp
+ movrz $len, 0, $inc
+
+ ldx [$inp], %o2
+ sllx %o0, $ileft, %o0
+ srlx %o1, $iright, %g1
+ sllx %o1, $ileft, %o1
+ or %g1, %o0, %o0
+ srlx %o2, $iright, %o2
+ or %o2, %o1, %o1
+
+.Lctr32_aligned_inp:
+ fmovd %f0, %f4
+ faesencx %f2, %f6, %f0
+ faesencx %f4, %f8, %f2
+ ldd [$key + 64], %f6 ! round[4]
+ ldd [$key + 72], %f8
+ add $key, 64, $end
+ sub $rounds, 16*8, $inner
+
+ stx %o0, [%sp + LOCALS + 0]
+ stx %o1, [%sp + LOCALS + 8]
+ add $inp, $inc, $inp ! inp+=16
+
+.Lctr32_enc_unaligned:
+ fmovd %f0, %f4
+ faesencx %f2, %f10, %f0
+ faesencx %f4, %f12, %f2
+ ldd [$end + 16], %f10
+ ldd [$end + 24], %f12
+ add $end, 32, $end
+
+ fmovd %f0, %f4
+ faesencx %f2, %f6, %f0
+ faesencx %f4, %f8, %f2
+ ldd [$end + 0], %f6
+ ldd [$end + 8], %f8
+
+ brnz,a $inner, .Lctr32_enc_unaligned
+ sub $inner, 16*2, $inner
+
+ fmovd %f0, %f4
+ faesencx %f2, %f10, %f0
+ faesencx %f4, %f12, %f2
+ ldd [$end + 16], %f10 ! round[last-1]
+ ldd [$end + 24], %f12
+ fpadd32 $ctr1, $one, $ctr1 ! increment counter
+
+ fmovd %f0, %f4
+ faesencx %f2, %f6, %f0
+ faesencx %f4, %f8, %f2
+ fxor $in0, $rlhi, %f6 ! inp^round[last]
+ fxor $in1, $rllo, %f8
+ ldd [%sp + LOCALS + 0], $in0
+ ldd [%sp + LOCALS + 8], $in1
+
+ fmovd %f0, %f4
+ faesencx %f2, %f10, %f0
+ faesencx %f4, %f12, %f2
+ ldd [$key + 16], %f10 ! round[1]
+ ldd [$key + 24], %f12
+
+ fmovd %f0, %f4
+ faesenclx %f2, %f6, %f0
+ faesenclx %f4, %f8, %f2
+
+ faligndata $outhead, %f0, $out0
+ faligndata %f0, %f2, $out1
+ std $out0, [$out + 0]
+ std $out1, [$out + 8]
+ add $out, 16, $out
+
+ brnz,a $len, .Loop_ctr32_unaligned_out
+ sub $len, 1, $len
+
+.Lctr32_unaligned_out_done:
+ faligndata %f2, %f2, %f8
+ stda %f8, [$out + $mask]0xc0 ! partial store
+
+ ret
+ restore
+.type aes_fx_ctr32_encrypt_blocks,#function
+.size aes_fx_ctr32_encrypt_blocks,.-aes_fx_ctr32_encrypt_blocks
+.align 32
+.Lone:
+ .word 0, 1
+.asciz "AES for Fujitsu SPARC64 X, CRYPTOGAMS by <appro\@openssl.org>"
+.align 4
+___
+}