+
+if ($flavour =~ /64/) { # 4x subroutine
+my ($I0,$j1,$j2,$j3,
+ $I1,$I2,$I3,$H3,$H34,$H4,$Yl,$Ym,$Yh) = map("q$_",(4..7,15..23));
+
+$code.=<<___;
+.type gcm_ghash_v8_4x,%function
+.align 4
+gcm_ghash_v8_4x:
+.Lgcm_ghash_v8_4x:
+ vld1.64 {$Xl},[$Xi] @ load [rotated] Xi
+ vld1.64 {$H-$H2},[$Htbl],#48 @ load twisted H, ..., H^2
+ vmov.i8 $xC2,#0xe1
+ vld1.64 {$H3-$H4},[$Htbl] @ load twisted H^3, ..., H^4
+ vshl.u64 $xC2,$xC2,#57 @ compose 0xc2.0 constant
+#ifndef __ARMEB__
+ vrev64.8 $Xl,$Xl
+#endif
+ b .Loop4x
+
+.align 4
+.Loop4x:
+ vld1.64 {$I0-$j3},[$inp],#64
+#ifndef __ARMEB__
+ vrev64.8 $j1,$j1
+ vrev64.8 $j2,$j2
+ vrev64.8 $j3,$j3
+ vrev64.8 $I0,$I0
+#endif
+ vext.8 $I3,$j3,$j3,#8
+ vext.8 $I2,$j2,$j2,#8
+ vext.8 $I1,$j1,$j1,#8
+
+ vpmull.p64 $Yl,$H,$I3 @ H·Ii+3
+ veor $j3,$j3,$I3
+ vpmull2.p64 $Yh,$H,$I3
+ vpmull.p64 $Ym,$Hhl,$j3
+
+ vpmull.p64 $t0,$H2,$I2 @ H^2·Ii+2
+ veor $j2,$j2,$I2
+ vpmull2.p64 $I2,$H2,$I2
+ vpmull2.p64 $j2,$Hhl,$j2
+
+ veor $Yl,$Yl,$t0
+ veor $Yh,$Yh,$I2
+ veor $Ym,$Ym,$j2
+
+ vpmull.p64 $j3,$H3,$I1 @ H^3·Ii+1
+ veor $j1,$j1,$I1
+ vpmull2.p64 $I1,$H3,$I1
+ vpmull.p64 $j1,$H34,$j1
+
+ veor $Yl,$Yl,$j3
+ veor $Yh,$Yh,$I1
+ veor $Ym,$Ym,$j1
+
+ veor $t0,$I0,$Xl
+ vext.8 $IN,$t0,$t0,#8
+
+ vpmull.p64 $Xl,$H4,$IN @ H^4·(Xi+Ii)
+ veor $t0,$t0,$IN
+ vpmull2.p64 $Xh,$H4,$IN
+ vpmull2.p64 $Xm,$H34,$t0
+
+ veor $Xl,$Xl,$Yl
+ veor $Xh,$Xh,$Yh
+ veor $Xm,$Xm,$Ym
+
+ vext.8 $t1,$Xl,$Xh,#8 @ Karatsuba post-processing
+ veor $t2,$Xl,$Xh
+ veor $Xm,$Xm,$t1
+ veor $Xm,$Xm,$t2
+
+ vpmull.p64 $t2,$Xl,$xC2 @ 1st phase of reduction
+ vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result
+ vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl
+ veor $Xl,$Xm,$t2
+
+ vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase of reduction
+ vpmull.p64 $Xl,$Xl,$xC2
+ veor $t2,$t2,$Xh
+ veor $Xl,$Xl,$t2
+ vext.8 $Xl,$Xl,$Xl,#8
+
+ subs $len,$len,#64
+ b.ne .Loop4x
+
+#ifndef __ARMEB__
+ vrev64.8 $Xl,$Xl
+#endif
+ vst1.64 {$Xl},[$Xi] @ write out Xi
+
+ ret
+.size gcm_ghash_v8_4x,.-gcm_ghash_v8_4x
+___
+