2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
17 # This module implements Poly1305 hash for ARMv8.
21 # Numbers are cycles per processed byte with poly1305_blocks alone.
25 # Apple A7 1.86/+5% 0.72
26 # Cortex-A53 2.69/+58% 1.47
27 # Cortex-A57 2.70/+7% 1.14
28 # Denver 1.64/+50% 1.18(*)
29 # X-Gene 2.13/+68% 2.27
30 # Mongoose 1.77/+75% 1.12
32 # (*) estimate based on resources availability is less than 1.0,
33 # i.e. measured result is worse than expected, presumably binary
34 # translator is not almighty;
39 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
40 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
41 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
42 die "can't locate arm-xlate.pl";
44 open OUT,"| \"$^X\" $xlate $flavour $output";
47 my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3));
48 my ($mac,$nonce)=($inp,$len);
50 my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14));
57 // forward "declarations" are required for Apple
58 .extern OPENSSL_armcap_P
59 .globl poly1305_blocks
63 .type poly1305_init,%function
67 stp xzr,xzr,[$ctx] // zero hash value
68 stp xzr,xzr,[$ctx,#16] // [along with is_base2_26]
74 ldrsw $t1,.LOPENSSL_armcap_P
76 ldr $t1,.LOPENSSL_armcap_P
78 adr $t0,.LOPENSSL_armcap_P
80 ldp $r0,$r1,[$inp] // load key
81 mov $s1,#0xfffffffc0fffffff
82 movk $s1,#0x0fff,lsl#48
85 rev $r0,$r0 // flip bytes
88 and $r0,$r0,$s1 // &=0ffffffc0fffffff
90 and $r1,$r1,$s1 // &=0ffffffc0ffffffc
91 stp $r0,$r1,[$ctx,#32] // save key value
95 adr $d0,poly1305_blocks
96 adr $r0,poly1305_blocks_neon
98 adr $r1,poly1305_emit_neon
108 .size poly1305_init,.-poly1305_init
110 .type poly1305_blocks,%function
116 ldp $h0,$h1,[$ctx] // load hash value
117 ldp $r0,$r1,[$ctx,#32] // load key value
119 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
124 ldp $t0,$t1,[$inp],#16 // load input
130 adds $h0,$h0,$t0 // accumulate input
133 mul $d0,$h0,$r0 // h0*r0
137 mul $t0,$h1,$s1 // h1*5*r1
141 mul $t0,$h0,$r1 // h0*r1
146 mul $t0,$h1,$r0 // h1*r0
151 mul $t0,$h2,$s1 // h2*5*r1
153 mul $t1,$h2,$r0 // h2*r0
158 and $t0,$d2,#-4 // final reduction
160 add $t0,$t0,$d2,lsr#2
167 stp $h0,$h1,[$ctx] // store hash value
172 .size poly1305_blocks,.-poly1305_blocks
174 .type poly1305_emit,%function
177 ldp $h0,$h1,[$ctx] // load hash base 2^64
179 ldp $t0,$t1,[$nonce] // load nonce
181 adds $d0,$h0,#5 // compare to modulus
185 tst $d2,#-4 // see if it's carried/borrowed
191 ror $t0,$t0,#32 // flip nonce words
194 adds $h0,$h0,$t0 // accumulate nonce
197 rev $h0,$h0 // flip output bytes
200 stp $h0,$h1,[$mac] // write result
203 .size poly1305_emit,.-poly1305_emit
205 my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8));
206 my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13));
207 my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18));
208 my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23));
209 my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28));
210 my ($T0,$T1,$MASK) = map("v$_",(29..31));
212 my ($in2,$zeros)=("x16","x17");
213 my $is_base2_26 = $zeros; # borrow
216 .type poly1305_mult,%function
219 mul $d0,$h0,$r0 // h0*r0
222 mul $t0,$h1,$s1 // h1*5*r1
226 mul $t0,$h0,$r1 // h0*r1
231 mul $t0,$h1,$r0 // h1*r0
236 mul $t0,$h2,$s1 // h2*5*r1
238 mul $t1,$h2,$r0 // h2*r0
243 and $t0,$d2,#-4 // final reduction
245 add $t0,$t0,$d2,lsr#2
251 .size poly1305_mult,.-poly1305_mult
253 .type poly1305_splat,%function
256 and x12,$h0,#0x03ffffff // base 2^64 -> base 2^26
259 and x14,x14,#0x03ffffff
263 str w12,[$ctx,#16*0] // r0
264 add w12,w13,w13,lsl#2 // r1*5
265 str w13,[$ctx,#16*1] // r1
266 add w13,w14,w14,lsl#2 // r2*5
267 str w12,[$ctx,#16*2] // s1
268 str w14,[$ctx,#16*3] // r2
269 add w14,w15,w15,lsl#2 // r3*5
270 str w13,[$ctx,#16*4] // s2
271 str w15,[$ctx,#16*5] // r3
272 add w15,w16,w16,lsl#2 // r4*5
273 str w14,[$ctx,#16*6] // s3
274 str w16,[$ctx,#16*7] // r4
275 str w15,[$ctx,#16*8] // s4
278 .size poly1305_splat,.-poly1305_splat
280 .type poly1305_blocks_neon,%function
282 poly1305_blocks_neon:
283 ldr $is_base2_26,[$ctx,#24]
286 cbz $is_base2_26,poly1305_blocks
289 stp x29,x30,[sp,#-80]!
295 cbz $is_base2_26,.Lbase2_64_neon
297 ldp w10,w11,[$ctx] // load hash value base 2^26
298 ldp w12,w13,[$ctx,#8]
304 ldp $r0,$r1,[$ctx,#32] // load key value
306 add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64
308 adds $h0,$h0,x12,lsl#52
309 add $h1,$h1,x13,lsl#14
312 adds $h1,$h1,x14,lsl#40
313 adc $d2,$h2,xzr // can be partially reduced...
315 ldp $d0,$d1,[$inp],#16 // load input
317 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
319 and $t0,$d2,#-4 // ... so reduce
321 add $t0,$t0,$d2,lsr#2
330 adds $h0,$h0,$d0 // accumulate input
337 cbz $padbit,.Lstore_base2_64_neon
339 and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26
342 and x12,x12,#0x03ffffff
346 cbnz $len,.Leven_neon
348 stp w10,w11,[$ctx] // store hash value base 2^26
349 stp w12,w13,[$ctx,#8]
354 .Lstore_base2_64_neon:
355 stp $h0,$h1,[$ctx] // store hash value base 2^64
356 stp $h2,xzr,[$ctx,#16] // note that is_base2_26 is zeroed
361 ldp $r0,$r1,[$ctx,#32] // load key value
363 ldp $h0,$h1,[$ctx] // load hash value base 2^64
369 ldp $d0,$d1,[$inp],#16 // load input
371 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
376 adds $h0,$h0,$d0 // accumulate input
383 and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26
386 and x12,x12,#0x03ffffff
390 stp d8,d9,[sp,#16] // meet ABI requirements
401 ////////////////////////////////// initialize r^n table
403 add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
409 bl poly1305_mult // r^2
413 bl poly1305_mult // r^3
417 bl poly1305_mult // r^4
425 csel $in2,$zeros,$in2,lo
428 str x4,[$ctx,#-24] // set is_base2_26
429 sub $ctx,$ctx,#48 // restore original $ctx
437 csel $in2,$zeros,$in2,lo
439 stp d8,d9,[sp,#16] // meet ABI requirements
451 ldp x8,x12,[$in2],#16 // inp[2:3] (or zero)
452 ldp x9,x13,[$in2],#48
454 lsl $padbit,$padbit,#24
463 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
464 and x5,x9,#0x03ffffff
467 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
470 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
472 and x8,x8,#0x03ffffff
473 and x9,x9,#0x03ffffff
476 add x12,$padbit,x12,lsr#40
477 add x13,$padbit,x13,lsr#40
478 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
480 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
481 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
486 ldp x8,x12,[$inp],#16 // inp[0:1]
487 ldp x9,x13,[$inp],#48
489 ld1 {$R0,$R1,$S1,$R2},[x15],#64
490 ld1 {$S2,$R3,$S3,$R4},[x15],#64
499 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
500 and x5,x9,#0x03ffffff
503 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
506 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
508 and x8,x8,#0x03ffffff
509 and x9,x9,#0x03ffffff
512 add x12,$padbit,x12,lsr#40
513 add x13,$padbit,x13,lsr#40
514 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
516 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
517 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
522 ushr $MASK.2d,$MASK.2d,#38
528 ////////////////////////////////////////////////////////////////
529 // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
530 // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
531 // \___________________/
532 // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
533 // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
534 // \___________________/ \____________________/
536 // Note that we start with inp[2:3]*r^2. This is because it
537 // doesn't depend on reduction in previous iteration.
538 ////////////////////////////////////////////////////////////////
539 // d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0
540 // d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4
541 // d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3
542 // d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2
543 // d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
546 umull $ACC4,$IN23_0,${R4}[2]
547 csel $in2,$zeros,$in2,lo
548 umull $ACC3,$IN23_0,${R3}[2]
549 umull $ACC2,$IN23_0,${R2}[2]
550 ldp x8,x12,[$in2],#16 // inp[2:3] (or zero)
551 umull $ACC1,$IN23_0,${R1}[2]
552 ldp x9,x13,[$in2],#48
553 umull $ACC0,$IN23_0,${R0}[2]
561 umlal $ACC4,$IN23_1,${R3}[2]
562 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
563 umlal $ACC3,$IN23_1,${R2}[2]
564 and x5,x9,#0x03ffffff
565 umlal $ACC2,$IN23_1,${R1}[2]
567 umlal $ACC1,$IN23_1,${R0}[2]
569 umlal $ACC0,$IN23_1,${S4}[2]
570 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
572 umlal $ACC4,$IN23_2,${R2}[2]
574 umlal $ACC3,$IN23_2,${R1}[2]
576 umlal $ACC2,$IN23_2,${R0}[2]
577 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
578 umlal $ACC1,$IN23_2,${S4}[2]
580 umlal $ACC0,$IN23_2,${S3}[2]
581 and x8,x8,#0x03ffffff
583 umlal $ACC4,$IN23_3,${R1}[2]
584 and x9,x9,#0x03ffffff
585 umlal $ACC3,$IN23_3,${R0}[2]
587 umlal $ACC2,$IN23_3,${S4}[2]
589 umlal $ACC1,$IN23_3,${S3}[2]
590 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
591 umlal $ACC0,$IN23_3,${S2}[2]
594 add $IN01_2,$IN01_2,$H2
595 add x12,$padbit,x12,lsr#40
596 umlal $ACC4,$IN23_4,${R0}[2]
597 add x13,$padbit,x13,lsr#40
598 umlal $ACC3,$IN23_4,${S4}[2]
599 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
600 umlal $ACC2,$IN23_4,${S3}[2]
601 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
602 umlal $ACC1,$IN23_4,${S2}[2]
604 umlal $ACC0,$IN23_4,${S1}[2]
607 ////////////////////////////////////////////////////////////////
608 // (hash+inp[0:1])*r^4 and accumulate
610 add $IN01_0,$IN01_0,$H0
612 umlal $ACC3,$IN01_2,${R1}[0]
613 ldp x8,x12,[$inp],#16 // inp[0:1]
614 umlal $ACC0,$IN01_2,${S3}[0]
615 ldp x9,x13,[$inp],#48
616 umlal $ACC4,$IN01_2,${R2}[0]
617 umlal $ACC1,$IN01_2,${S4}[0]
618 umlal $ACC2,$IN01_2,${R0}[0]
626 add $IN01_1,$IN01_1,$H1
627 umlal $ACC3,$IN01_0,${R3}[0]
628 umlal $ACC4,$IN01_0,${R4}[0]
629 and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
630 umlal $ACC2,$IN01_0,${R2}[0]
631 and x5,x9,#0x03ffffff
632 umlal $ACC0,$IN01_0,${R0}[0]
634 umlal $ACC1,$IN01_0,${R1}[0]
637 add $IN01_3,$IN01_3,$H3
638 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
639 umlal $ACC3,$IN01_1,${R2}[0]
641 umlal $ACC4,$IN01_1,${R3}[0]
643 umlal $ACC0,$IN01_1,${S4}[0]
644 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
645 umlal $ACC2,$IN01_1,${R1}[0]
647 umlal $ACC1,$IN01_1,${R0}[0]
648 and x8,x8,#0x03ffffff
650 add $IN01_4,$IN01_4,$H4
651 and x9,x9,#0x03ffffff
652 umlal $ACC3,$IN01_3,${R0}[0]
654 umlal $ACC0,$IN01_3,${S2}[0]
656 umlal $ACC4,$IN01_3,${R1}[0]
657 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
658 umlal $ACC1,$IN01_3,${S3}[0]
660 umlal $ACC2,$IN01_3,${S4}[0]
661 add x12,$padbit,x12,lsr#40
663 umlal $ACC3,$IN01_4,${S4}[0]
664 add x13,$padbit,x13,lsr#40
665 umlal $ACC0,$IN01_4,${S1}[0]
666 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
667 umlal $ACC4,$IN01_4,${R0}[0]
668 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
669 umlal $ACC1,$IN01_4,${S2}[0]
671 umlal $ACC2,$IN01_4,${S3}[0]
675 /////////////////////////////////////////////////////////////////
676 // lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
679 // [see discussion in poly1305-armv4 module]
681 ushr $T0.2d,$ACC3,#26
683 ushr $T1.2d,$ACC0,#26
684 and $ACC0,$ACC0,$MASK.2d
685 add $ACC4,$ACC4,$T0.2d // h3 -> h4
686 bic $H3,#0xfc,lsl#24 // &=0x03ffffff
687 add $ACC1,$ACC1,$T1.2d // h0 -> h1
689 ushr $T0.2d,$ACC4,#26
691 ushr $T1.2d,$ACC1,#26
694 add $ACC2,$ACC2,$T1.2d // h1 -> h2
696 add $ACC0,$ACC0,$T0.2d
698 shrn $T1.2s,$ACC2,#26
700 add $ACC0,$ACC0,$T0.2d // h4 -> h0
702 add $H3,$H3,$T1.2s // h2 -> h3
705 shrn $T0.2s,$ACC0,#26
710 add $H1,$H1,$T0.2s // h0 -> h1
711 add $H4,$H4,$T1.2s // h3 -> h4
716 dup $IN23_2,${IN23_2}[0]
717 add $IN01_2,$IN01_2,$H2
719 ////////////////////////////////////////////////////////////////
720 // multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
725 dup $IN23_2,${IN01_2}[0]
726 add $IN23_0,$IN01_0,$H0
727 add $IN23_3,$IN01_3,$H3
728 add $IN23_1,$IN01_1,$H1
729 add $IN23_4,$IN01_4,$H4
732 dup $IN23_0,${IN23_0}[0]
733 umull2 $ACC0,$IN23_2,${S3}
734 umull2 $ACC3,$IN23_2,${R1}
735 umull2 $ACC4,$IN23_2,${R2}
736 umull2 $ACC2,$IN23_2,${R0}
737 umull2 $ACC1,$IN23_2,${S4}
739 dup $IN23_1,${IN23_1}[0]
740 umlal2 $ACC0,$IN23_0,${R0}
741 umlal2 $ACC2,$IN23_0,${R2}
742 umlal2 $ACC3,$IN23_0,${R3}
743 umlal2 $ACC4,$IN23_0,${R4}
744 umlal2 $ACC1,$IN23_0,${R1}
746 dup $IN23_3,${IN23_3}[0]
747 umlal2 $ACC0,$IN23_1,${S4}
748 umlal2 $ACC3,$IN23_1,${R2}
749 umlal2 $ACC2,$IN23_1,${R1}
750 umlal2 $ACC4,$IN23_1,${R3}
751 umlal2 $ACC1,$IN23_1,${R0}
753 dup $IN23_4,${IN23_4}[0]
754 umlal2 $ACC3,$IN23_3,${R0}
755 umlal2 $ACC4,$IN23_3,${R1}
756 umlal2 $ACC0,$IN23_3,${S2}
757 umlal2 $ACC1,$IN23_3,${S3}
758 umlal2 $ACC2,$IN23_3,${S4}
760 umlal2 $ACC3,$IN23_4,${S4}
761 umlal2 $ACC0,$IN23_4,${S1}
762 umlal2 $ACC4,$IN23_4,${R0}
763 umlal2 $ACC1,$IN23_4,${S2}
764 umlal2 $ACC2,$IN23_4,${S3}
768 ////////////////////////////////////////////////////////////////
769 // (hash+inp[0:1])*r^4:r^3 and accumulate
771 add $IN01_0,$IN01_0,$H0
772 umlal $ACC3,$IN01_2,${R1}
773 umlal $ACC0,$IN01_2,${S3}
774 umlal $ACC4,$IN01_2,${R2}
775 umlal $ACC1,$IN01_2,${S4}
776 umlal $ACC2,$IN01_2,${R0}
778 add $IN01_1,$IN01_1,$H1
779 umlal $ACC3,$IN01_0,${R3}
780 umlal $ACC0,$IN01_0,${R0}
781 umlal $ACC4,$IN01_0,${R4}
782 umlal $ACC1,$IN01_0,${R1}
783 umlal $ACC2,$IN01_0,${R2}
785 add $IN01_3,$IN01_3,$H3
786 umlal $ACC3,$IN01_1,${R2}
787 umlal $ACC0,$IN01_1,${S4}
788 umlal $ACC4,$IN01_1,${R3}
789 umlal $ACC1,$IN01_1,${R0}
790 umlal $ACC2,$IN01_1,${R1}
792 add $IN01_4,$IN01_4,$H4
793 umlal $ACC3,$IN01_3,${R0}
794 umlal $ACC0,$IN01_3,${S2}
795 umlal $ACC4,$IN01_3,${R1}
796 umlal $ACC1,$IN01_3,${S3}
797 umlal $ACC2,$IN01_3,${S4}
799 umlal $ACC3,$IN01_4,${S4}
800 umlal $ACC0,$IN01_4,${S1}
801 umlal $ACC4,$IN01_4,${R0}
802 umlal $ACC1,$IN01_4,${S2}
803 umlal $ACC2,$IN01_4,${S3}
806 ////////////////////////////////////////////////////////////////
809 addp $ACC3,$ACC3,$ACC3
810 ldp d8,d9,[sp,#16] // meet ABI requirements
811 addp $ACC0,$ACC0,$ACC0
813 addp $ACC4,$ACC4,$ACC4
815 addp $ACC1,$ACC1,$ACC1
817 addp $ACC2,$ACC2,$ACC2
819 ////////////////////////////////////////////////////////////////
820 // lazy reduction, but without narrowing
822 ushr $T0.2d,$ACC3,#26
823 and $ACC3,$ACC3,$MASK.2d
824 ushr $T1.2d,$ACC0,#26
825 and $ACC0,$ACC0,$MASK.2d
827 add $ACC4,$ACC4,$T0.2d // h3 -> h4
828 add $ACC1,$ACC1,$T1.2d // h0 -> h1
830 ushr $T0.2d,$ACC4,#26
831 and $ACC4,$ACC4,$MASK.2d
832 ushr $T1.2d,$ACC1,#26
833 and $ACC1,$ACC1,$MASK.2d
834 add $ACC2,$ACC2,$T1.2d // h1 -> h2
836 add $ACC0,$ACC0,$T0.2d
838 ushr $T1.2d,$ACC2,#26
839 and $ACC2,$ACC2,$MASK.2d
840 add $ACC0,$ACC0,$T0.2d // h4 -> h0
841 add $ACC3,$ACC3,$T1.2d // h2 -> h3
843 ushr $T0.2d,$ACC0,#26
844 and $ACC0,$ACC0,$MASK.2d
845 ushr $T1.2d,$ACC3,#26
846 and $ACC3,$ACC3,$MASK.2d
847 add $ACC1,$ACC1,$T0.2d // h0 -> h1
848 add $ACC4,$ACC4,$T1.2d // h3 -> h4
850 ////////////////////////////////////////////////////////////////
851 // write the result, can be partially reduced
853 st4 {$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16
854 st1 {$ACC4}[0],[$ctx]
859 .size poly1305_blocks_neon,.-poly1305_blocks_neon
861 .type poly1305_emit_neon,%function
864 ldr $is_base2_26,[$ctx,#24]
865 cbz $is_base2_26,poly1305_emit
867 ldp w10,w11,[$ctx] // load hash value base 2^26
868 ldp w12,w13,[$ctx,#8]
871 add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64
873 adds $h0,$h0,x12,lsl#52
874 add $h1,$h1,x13,lsl#14
877 adds $h1,$h1,x14,lsl#40
878 adc $h2,$h2,xzr // can be partially reduced...
880 ldp $t0,$t1,[$nonce] // load nonce
882 and $d0,$h2,#-4 // ... so reduce
883 add $d0,$d0,$h2,lsr#2
889 adds $d0,$h0,#5 // compare to modulus
893 tst $d2,#-4 // see if it's carried/borrowed
899 ror $t0,$t0,#32 // flip nonce words
902 adds $h0,$h0,$t0 // accumulate nonce
905 rev $h0,$h0 // flip output bytes
908 stp $h0,$h1,[$mac] // write result
911 .size poly1305_emit_neon,.-poly1305_emit_neon
915 .long 0,0,0,0,0,0,0,0
918 .long OPENSSL_armcap_P-.
920 .quad OPENSSL_armcap_P-.
922 .asciz "Poly1305 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
926 foreach (split("\n",$code)) {
927 s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/ or
928 s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/ or
929 (m/\bdup\b/ and (s/\.[24]s/.2d/g or 1)) or
930 (m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1)) or
931 (m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1)) or
932 (m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1)) or
933 (m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1));
935 s/\.[124]([sd])\[/.$1\[/;