3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
10 # This module implements Poly1305 hash for ARMv8.
14 # Numbers are cycles per processed byte with poly1305_blocks alone.
18 # Apple A7 1.86/+5% 0.72
19 # Cortex-A53 2.63/+58% 1.47
20 # Cortex-A57 2.70/+7% 1.14
21 # Denver 1.39/+50% 1.18(*)
22 # X-Gene 2.00/+68% 2.19
24 # (*) estimate based on resources availability is less than 1.0,
25 # i.e. measured result is worse than expected, presumably binary
26 # translator is not almighty;
31 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
32 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
33 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
34 die "can't locate arm-xlate.pl";
36 open OUT,"| \"$^X\" $xlate $flavour $output";
39 my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3));
40 my ($mac,$nonce)=($inp,$len);
42 my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14));
49 // forward "declarations" are required for Apple
50 .extern OPENSSL_armcap_P
51 .globl poly1305_blocks
55 .type poly1305_init,%function
59 stp xzr,xzr,[$ctx] // zero hash value
60 stp xzr,xzr,[$ctx,#16] // [along with is_base2_26]
66 ldrsw $t1,.LOPENSSL_armcap_P
68 ldr $t1,.LOPENSSL_armcap_P
70 adr $t0,.LOPENSSL_armcap_P
72 ldp $r0,$r1,[$inp] // load key
73 mov $s1,#0xfffffffc0fffffff
74 movk $s1,#0x0fff,lsl#48
77 rev $r0,$r0 // flip bytes
80 and $r0,$r0,$s1 // &=0ffffffc0fffffff
82 and $r1,$r1,$s1 // &=0ffffffc0ffffffc
83 stp $r0,$r1,[$ctx,#32] // save key value
87 adr $d0,poly1305_blocks
88 adr $r0,poly1305_blocks_neon
90 adr $r1,poly1305_emit_neon
100 .size poly1305_init,.-poly1305_init
102 .type poly1305_blocks,%function
108 ldp $h0,$h1,[$ctx] // load hash value
109 ldp $r0,$r1,[$ctx,#32] // load key value
111 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
116 ldp $t0,$t1,[$inp],#16 // load input
122 adds $h0,$h0,$t0 // accumulate input
125 mul $d0,$h0,$r0 // h0*r0
129 mul $t0,$h1,$s1 // h1*5*r1
133 mul $t0,$h0,$r1 // h0*r1
138 mul $t0,$h1,$r0 // h1*r0
143 mul $t0,$h2,$s1 // h2*5*r1
145 mul $t1,$h2,$r0 // h2*r0
150 and $t0,$d2,#-4 // final reduction
152 add $t0,$t0,$d2,lsr#2
158 stp $h0,$h1,[$ctx] // store hash value
163 .size poly1305_blocks,.-poly1305_blocks
165 .type poly1305_emit,%function
168 ldp $h0,$h1,[$ctx] // load hash base 2^64
170 ldp $t0,$t1,[$nonce] // load nonce
172 adds $d0,$h0,#5 // compare to modulus
176 tst $d2,#-4 // see if it's carried/borrowed
182 ror $t0,$t0,#32 // flip nonce words
185 adds $h0,$h0,$t0 // accumulate nonce
188 rev $h0,$h0 // flip output bytes
191 stp $h0,$h1,[$mac] // write result
194 .size poly1305_emit,.-poly1305_emit
196 my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8));
197 my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13));
198 my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18));
199 my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23));
200 my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28));
201 my ($T0,$T1,$MASK) = map("v$_",(29..31));
203 my ($in2,$zeros)=("x16","x17");
204 my $is_base2_26 = $zeros; # borrow
207 .type poly1305_mult,%function
210 mul $d0,$h0,$r0 // h0*r0
213 mul $t0,$h1,$s1 // h1*5*r1
217 mul $t0,$h0,$r1 // h0*r1
222 mul $t0,$h1,$r0 // h1*r0
227 mul $t0,$h2,$s1 // h2*5*r1
229 mul $t1,$h2,$r0 // h2*r0
234 and $t0,$d2,#-4 // final reduction
236 add $t0,$t0,$d2,lsr#2
241 .size poly1305_mult,.-poly1305_mult
243 .type poly1305_splat,%function
246 and x12,$h0,#0x03ffffff // base 2^64 -> base 2^26
249 and x14,x14,#0x03ffffff
253 str w12,[$ctx,#16*0] // r0
254 add w12,w13,w13,lsl#2 // r1*5
255 str w13,[$ctx,#16*1] // r1
256 add w13,w14,w14,lsl#2 // r2*5
257 str w12,[$ctx,#16*2] // s1
258 str w14,[$ctx,#16*3] // r2
259 add w14,w15,w15,lsl#2 // r3*5
260 str w13,[$ctx,#16*4] // s2
261 str w15,[$ctx,#16*5] // r3
262 add w15,w16,w16,lsl#2 // r4*5
263 str w14,[$ctx,#16*6] // s3
264 str w16,[$ctx,#16*7] // r4
265 str w15,[$ctx,#16*8] // s4
268 .size poly1305_splat,.-poly1305_splat
270 .type poly1305_blocks_neon,%function
272 poly1305_blocks_neon:
273 ldr $is_base2_26,[$ctx,#24]
276 cbz $is_base2_26,poly1305_blocks
279 stp x29,x30,[sp,#-80]!
285 cbz $is_base2_26,.Lbase2_64_neon
287 ldp w10,w11,[$ctx] // load hash value base 2^26
288 ldp w12,w13,[$ctx,#8]
294 ldp $r0,$r1,[$ctx,#32] // load key value
296 add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64
298 adds $h0,$h0,x12,lsl#52
299 add $h1,$h1,x13,lsl#14
302 adds $h1,$h1,x14,lsl#40
303 adc $d2,$h2,xzr // can be partially reduced...
305 ldp $d0,$d1,[$inp],#16 // load input
307 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
309 and $t0,$d2,#-4 // ... so reduce
311 add $t0,$t0,$d2,lsr#2
319 adds $h0,$h0,$d0 // accumulate input
326 cbz $padbit,.Lstore_base2_64_neon
328 and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26
331 and x12,x12,#0x03ffffff
335 cbnz $len,.Leven_neon
337 stp w10,w11,[$ctx] // store hash value base 2^26
338 stp w12,w13,[$ctx,#8]
343 .Lstore_base2_64_neon:
344 stp $h0,$h1,[$ctx] // store hash value base 2^64
345 stp $h2,xzr,[$ctx,#16] // note that is_base2_26 is zeroed
350 ldp $r0,$r1,[$ctx,#32] // load key value
352 ldp $h0,$h1,[$ctx] // load hash value base 2^64
358 ldp $d0,$d1,[$inp],#16 // load input
360 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
365 adds $h0,$h0,$d0 // accumulate input
372 and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26
375 and x12,x12,#0x03ffffff
379 stp d8,d9,[sp,#16] // meet ABI requirements
390 ////////////////////////////////// initialize r^n table
392 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
398 bl poly1305_mult // r^2
402 bl poly1305_mult // r^3
406 bl poly1305_mult // r^4
414 csel $in2,$zeros,$in2,lo
417 str x4,[$ctx,#-24] // set is_base2_26
418 sub $ctx,$ctx,#48 // restore original $ctx
426 csel $in2,$zeros,$in2,lo
428 stp d8,d9,[sp,#16] // meet ABI requirements
440 ldp x8,x12,[$in2],#16 // inp[2:3] (or zero)
441 ldp x9,x13,[$in2],#48
443 lsl $padbit,$padbit,#24
452 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
453 and x5,x9,#0x03ffffff
456 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
459 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
461 and x8,x8,#0x03ffffff
462 and x9,x9,#0x03ffffff
465 add x12,$padbit,x12,lsr#40
466 add x13,$padbit,x13,lsr#40
467 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
469 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
470 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
475 ldp x8,x12,[$inp],#16 // inp[0:1]
476 ldp x9,x13,[$inp],#48
478 ld1 {$R0,$R1,$S1,$R2},[x15],#64
479 ld1 {$S2,$R3,$S3,$R4},[x15],#64
488 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
489 and x5,x9,#0x03ffffff
492 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
495 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
497 and x8,x8,#0x03ffffff
498 and x9,x9,#0x03ffffff
501 add x12,$padbit,x12,lsr#40
502 add x13,$padbit,x13,lsr#40
503 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
505 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
506 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
515 ////////////////////////////////////////////////////////////////
516 // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
517 // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
518 // \___________________/
519 // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
520 // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
521 // \___________________/ \____________________/
523 // Note that we start with inp[2:3]*r^2. This is because it
524 // doesn't depend on reduction in previous iteration.
525 ////////////////////////////////////////////////////////////////
526 // d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0
527 // d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4
528 // d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3
529 // d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2
530 // d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
533 umull $ACC4,$IN23_0,${R4}[2]
534 csel $in2,$zeros,$in2,lo
535 umull $ACC3,$IN23_0,${R3}[2]
536 umull $ACC2,$IN23_0,${R2}[2]
537 ldp x8,x12,[$in2],#16 // inp[2:3] (or zero)
538 umull $ACC1,$IN23_0,${R1}[2]
539 ldp x9,x13,[$in2],#48
540 umull $ACC0,$IN23_0,${R0}[2]
548 umlal $ACC4,$IN23_1,${R3}[2]
549 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
550 umlal $ACC3,$IN23_1,${R2}[2]
551 and x5,x9,#0x03ffffff
552 umlal $ACC2,$IN23_1,${R1}[2]
554 umlal $ACC1,$IN23_1,${R0}[2]
556 umlal $ACC0,$IN23_1,${S4}[2]
557 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
559 umlal $ACC4,$IN23_2,${R2}[2]
561 umlal $ACC3,$IN23_2,${R1}[2]
563 umlal $ACC2,$IN23_2,${R0}[2]
564 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
565 umlal $ACC1,$IN23_2,${S4}[2]
567 umlal $ACC0,$IN23_2,${S3}[2]
568 and x8,x8,#0x03ffffff
570 umlal $ACC4,$IN23_3,${R1}[2]
571 and x9,x9,#0x03ffffff
572 umlal $ACC3,$IN23_3,${R0}[2]
574 umlal $ACC2,$IN23_3,${S4}[2]
576 umlal $ACC1,$IN23_3,${S3}[2]
577 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
578 umlal $ACC0,$IN23_3,${S2}[2]
581 add $IN01_2,$IN01_2,$H2
582 add x12,$padbit,x12,lsr#40
583 umlal $ACC4,$IN23_4,${R0}[2]
584 add x13,$padbit,x13,lsr#40
585 umlal $ACC3,$IN23_4,${S4}[2]
586 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
587 umlal $ACC2,$IN23_4,${S3}[2]
588 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
589 umlal $ACC1,$IN23_4,${S2}[2]
591 umlal $ACC0,$IN23_4,${S1}[2]
594 ////////////////////////////////////////////////////////////////
595 // (hash+inp[0:1])*r^4 and accumulate
597 add $IN01_0,$IN01_0,$H0
599 umlal $ACC3,$IN01_2,${R1}[0]
600 ldp x8,x12,[$inp],#16 // inp[0:1]
601 umlal $ACC0,$IN01_2,${S3}[0]
602 ldp x9,x13,[$inp],#48
603 umlal $ACC4,$IN01_2,${R2}[0]
604 umlal $ACC1,$IN01_2,${S4}[0]
605 umlal $ACC2,$IN01_2,${R0}[0]
613 add $IN01_1,$IN01_1,$H1
614 umlal $ACC3,$IN01_0,${R3}[0]
615 umlal $ACC4,$IN01_0,${R4}[0]
616 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
617 umlal $ACC2,$IN01_0,${R2}[0]
618 and x5,x9,#0x03ffffff
619 umlal $ACC0,$IN01_0,${R0}[0]
621 umlal $ACC1,$IN01_0,${R1}[0]
624 add $IN01_3,$IN01_3,$H3
625 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
626 umlal $ACC3,$IN01_1,${R2}[0]
628 umlal $ACC4,$IN01_1,${R3}[0]
630 umlal $ACC0,$IN01_1,${S4}[0]
631 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
632 umlal $ACC2,$IN01_1,${R1}[0]
634 umlal $ACC1,$IN01_1,${R0}[0]
635 and x8,x8,#0x03ffffff
637 add $IN01_4,$IN01_4,$H4
638 and x9,x9,#0x03ffffff
639 umlal $ACC3,$IN01_3,${R0}[0]
641 umlal $ACC0,$IN01_3,${S2}[0]
643 umlal $ACC4,$IN01_3,${R1}[0]
644 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
645 umlal $ACC1,$IN01_3,${S3}[0]
647 umlal $ACC2,$IN01_3,${S4}[0]
648 add x12,$padbit,x12,lsr#40
650 umlal $ACC3,$IN01_4,${S4}[0]
651 add x13,$padbit,x13,lsr#40
652 umlal $ACC0,$IN01_4,${S1}[0]
653 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
654 umlal $ACC4,$IN01_4,${R0}[0]
655 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
656 umlal $ACC1,$IN01_4,${S2}[0]
658 umlal $ACC2,$IN01_4,${S3}[0]
661 /////////////////////////////////////////////////////////////////
662 // lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
665 ushr $T0.2d,$ACC3,#26
668 ushr $T1.2d,$ACC0,#26
670 add $ACC4,$ACC4,$T0.2d // h3 -> h4
671 bic $H3,#0xfc,lsl#24 // &=0x03ffffff
672 add $ACC1,$ACC1,$T1.2d // h0 -> h1
675 shrn $T0.2s,$ACC4,#26
677 ushr $T1.2d,$ACC1,#26
679 add $ACC2,$ACC2,$T1.2d // h1 -> h2
685 shrn $T1.2s,$ACC2,#26
687 add $H0,$H0,$T0.2s // h4 -> h0
688 add $H3,$H3,$T1.2s // h2 -> h3
695 add $H1,$H1,$T0.2s // h0 -> h1
696 add $H4,$H4,$T1.2s // h3 -> h4
701 dup $IN23_2,${IN23_2}[0]
703 add $IN01_2,$IN01_2,$H2
704 ushr $MASK.2d,$MASK.2d,#38
706 ////////////////////////////////////////////////////////////////
707 // multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
712 dup $IN23_2,${IN01_2}[0]
713 add $IN23_0,$IN01_0,$H0
714 add $IN23_3,$IN01_3,$H3
715 add $IN23_1,$IN01_1,$H1
716 add $IN23_4,$IN01_4,$H4
719 dup $IN23_0,${IN23_0}[0]
720 umull2 $ACC0,$IN23_2,${S3}
721 umull2 $ACC3,$IN23_2,${R1}
722 umull2 $ACC4,$IN23_2,${R2}
723 umull2 $ACC2,$IN23_2,${R0}
724 umull2 $ACC1,$IN23_2,${S4}
726 dup $IN23_1,${IN23_1}[0]
727 umlal2 $ACC0,$IN23_0,${R0}
728 umlal2 $ACC2,$IN23_0,${R2}
729 umlal2 $ACC3,$IN23_0,${R3}
730 umlal2 $ACC4,$IN23_0,${R4}
731 umlal2 $ACC1,$IN23_0,${R1}
733 dup $IN23_3,${IN23_3}[0]
734 umlal2 $ACC0,$IN23_1,${S4}
735 umlal2 $ACC3,$IN23_1,${R2}
736 umlal2 $ACC2,$IN23_1,${R1}
737 umlal2 $ACC4,$IN23_1,${R3}
738 umlal2 $ACC1,$IN23_1,${R0}
740 dup $IN23_4,${IN23_4}[0]
741 umlal2 $ACC3,$IN23_3,${R0}
742 umlal2 $ACC4,$IN23_3,${R1}
743 umlal2 $ACC0,$IN23_3,${S2}
744 umlal2 $ACC1,$IN23_3,${S3}
745 umlal2 $ACC2,$IN23_3,${S4}
747 umlal2 $ACC3,$IN23_4,${S4}
748 umlal2 $ACC0,$IN23_4,${S1}
749 umlal2 $ACC4,$IN23_4,${R0}
750 umlal2 $ACC1,$IN23_4,${S2}
751 umlal2 $ACC2,$IN23_4,${S3}
755 ////////////////////////////////////////////////////////////////
756 // (hash+inp[0:1])*r^4:r^3 and accumulate
758 add $IN01_0,$IN01_0,$H0
759 umlal $ACC3,$IN01_2,${R1}
760 umlal $ACC0,$IN01_2,${S3}
761 umlal $ACC4,$IN01_2,${R2}
762 umlal $ACC1,$IN01_2,${S4}
763 umlal $ACC2,$IN01_2,${R0}
765 add $IN01_1,$IN01_1,$H1
766 umlal $ACC3,$IN01_0,${R3}
767 umlal $ACC0,$IN01_0,${R0}
768 umlal $ACC4,$IN01_0,${R4}
769 umlal $ACC1,$IN01_0,${R1}
770 umlal $ACC2,$IN01_0,${R2}
772 add $IN01_3,$IN01_3,$H3
773 umlal $ACC3,$IN01_1,${R2}
774 umlal $ACC0,$IN01_1,${S4}
775 umlal $ACC4,$IN01_1,${R3}
776 umlal $ACC1,$IN01_1,${R0}
777 umlal $ACC2,$IN01_1,${R1}
779 add $IN01_4,$IN01_4,$H4
780 umlal $ACC3,$IN01_3,${R0}
781 umlal $ACC0,$IN01_3,${S2}
782 umlal $ACC4,$IN01_3,${R1}
783 umlal $ACC1,$IN01_3,${S3}
784 umlal $ACC2,$IN01_3,${S4}
786 umlal $ACC3,$IN01_4,${S4}
787 umlal $ACC0,$IN01_4,${S1}
788 umlal $ACC4,$IN01_4,${R0}
789 umlal $ACC1,$IN01_4,${S2}
790 umlal $ACC2,$IN01_4,${S3}
793 ////////////////////////////////////////////////////////////////
794 // lazy reduction, but without narrowing
796 ushr $T0.2d,$ACC3,#26
797 and $ACC3,$ACC3,$MASK.2d
798 ushr $T1.2d,$ACC0,#26
799 and $ACC0,$ACC0,$MASK.2d
801 add $ACC4,$ACC4,$T0.2d // h3 -> h4
802 add $ACC1,$ACC1,$T1.2d // h0 -> h1
804 ushr $T0.2d,$ACC4,#26
805 and $ACC4,$ACC4,$MASK.2d
806 ushr $T1.2d,$ACC1,#26
807 and $ACC1,$ACC1,$MASK.2d
808 add $ACC2,$ACC2,$T1.2d // h1 -> h2
810 add $ACC0,$ACC0,$T0.2d
812 ushr $T1.2d,$ACC2,#26
813 and $ACC2,$ACC2,$MASK.2d
814 add $ACC0,$ACC0,$T0.2d // h4 -> h0
815 add $ACC3,$ACC3,$T1.2d // h2 -> h3
817 ushr $T0.2d,$ACC0,#26
818 and $ACC0,$ACC0,$MASK.2d
819 ushr $T1.2d,$ACC3,#26
820 and $ACC3,$ACC3,$MASK.2d
821 add $ACC1,$ACC1,$T0.2d // h0 -> h1
822 add $ACC4,$ACC4,$T1.2d // h3 -> h4
824 ////////////////////////////////////////////////////////////////
827 addp $ACC2,$ACC2,$ACC2
828 ldp d8,d9,[sp,#16] // meet ABI requirements
829 addp $ACC0,$ACC0,$ACC0
831 addp $ACC1,$ACC1,$ACC1
833 addp $ACC3,$ACC3,$ACC3
835 addp $ACC4,$ACC4,$ACC4
837 ////////////////////////////////////////////////////////////////
838 // write the result, can be partially reduced
840 st4 {$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16
841 st1 {$ACC4}[0],[$ctx]
846 .size poly1305_blocks_neon,.-poly1305_blocks_neon
848 .type poly1305_emit_neon,%function
851 ldr $is_base2_26,[$ctx,#24]
852 cbz $is_base2_26,poly1305_emit
854 ldp w10,w11,[$ctx] // load hash value base 2^26
855 ldp w12,w13,[$ctx,#8]
858 add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64
860 adds $h0,$h0,x12,lsl#52
861 add $h1,$h1,x13,lsl#14
864 adds $h1,$h1,x14,lsl#40
865 adc $h2,$h2,xzr // can be partially reduced...
867 ldp $t0,$t1,[$nonce] // load nonce
869 and $d0,$h2,#-4 // ... so reduce
870 add $d0,$d0,$h2,lsr#2
875 adds $d0,$h0,#5 // compare to modulus
879 tst $d2,#-4 // see if it's carried/borrowed
885 ror $t0,$t0,#32 // flip nonce words
888 adds $h0,$h0,$t0 // accumulate nonce
891 rev $h0,$h0 // flip output bytes
894 stp $h0,$h1,[$mac] // write result
897 .size poly1305_emit_neon,.-poly1305_emit_neon
901 .long 0,0,0,0,0,0,0,0
904 .long OPENSSL_armcap_P-.
906 .quad OPENSSL_armcap_P-.
908 .asciz "Poly1305 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
912 foreach (split("\n",$code)) {
913 s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/ or
914 s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/ or
915 (m/\bdup\b/ and (s/\.[24]s/.2d/g or 1)) or
916 (m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1)) or
917 (m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1)) or
918 (m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1)) or
919 (m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1));
921 s/\.[124]([sd])\[/.$1\[/;