crypto/modes/asm/aes-gcm-armv8-unroll8_64.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License 2.0 (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9 #
  10 #========================================================================
  11 # Written by Xiaokang Qian <xiaokang.qian@arm.com> for the OpenSSL project,
  12 # derived from https://github.com/ARM-software/AArch64cryptolib, original
  13 # author Samuel Lee <Samuel.Lee@arm.com>. The module is, however, dual
  14 # licensed under OpenSSL and SPDX BSD-3-Clause licenses depending on where you
  15 # obtain it.
  16 #========================================================================
  17 #
  18 # Approach - We want to reload constants as we have plenty of spare ASIMD slots around crypto units for loading
  19 # Unroll x8 in main loop, main loop to act on 8 16B blocks per iteration, and then do modulo of the accumulated
  20 # intermediate hashesfrom the 8 blocks.
  21 #
  22 #  ____________________________________________________
  23 # |                                                    |
  24 # | PRE                                                |
  25 # |____________________________________________________|
  26 # |                |                |                  |
  27 # | CTR block 8k+13| AES block 8k+8 | GHASH block 8k+0 |
  28 # |________________|________________|__________________|
  29 # |                |                |                  |
  30 # | CTR block 8k+14| AES block 8k+9 | GHASH block 8k+1 |
  31 # |________________|________________|__________________|
  32 # |                |                |                  |
  33 # | CTR block 8k+15| AES block 8k+10| GHASH block 8k+2 |
  34 # |________________|________________|__________________|
  35 # |                |                |                  |
  36 # | CTR block 8k+16| AES block 8k+11| GHASH block 8k+3 |
  37 # |________________|________________|__________________|
  38 # |                |                |                  |
  39 # | CTR block 8k+17| AES block 8k+12| GHASH block 8k+4 |
  40 # |________________|________________|__________________|
  41 # |                |                |                  |
  42 # | CTR block 8k+18| AES block 8k+13| GHASH block 8k+5 |
  43 # |________________|________________|__________________|
  44 # |                |                |                  |
  45 # | CTR block 8k+19| AES block 8k+14| GHASH block 8k+6 |
  46 # |________________|________________|__________________|
  47 # |                |                |                  |
  48 # | CTR block 8k+20| AES block 8k+15| GHASH block 8k+7 |
  49 # |________________|____(mostly)____|__________________|
  50 # |                                                    |
  51 # | MODULO                                             |
  52 # |____________________________________________________|
  53 #
  54 # PRE:
  55 #     Ensure previous generated intermediate hash is aligned and merged with result for GHASH 4k+0
  56 # EXT low_acc, low_acc, low_acc, #8
  57 # EOR res_curr (8k+0), res_curr (4k+0), low_acc
  58 #
  59 # CTR block:
  60 #     Increment and byte reverse counter in scalar registers and transfer to SIMD registers
  61 # REV     ctr32, rev_ctr32
  62 # ORR     ctr64, constctr96_top32, ctr32, LSL #32
  63 # INS     ctr_next.d[0], constctr96_bottom64      // Keeping this in scalar registers to free up space in SIMD RF
  64 # INS     ctr_next.d[1], ctr64X
  65 # ADD     rev_ctr32, #1
  66 #
  67 # AES block:
  68 #      Do AES encryption/decryption on CTR block X and EOR it with input block X. Take 256 bytes key below for example.
  69 #      Doing small trick here of loading input in scalar registers, EORing with last key and then transferring
  70 #      Given we are very constrained in our ASIMD registers this is quite important
  71 #
  72 #      Encrypt:
  73 # LDR     input_low, [ input_ptr  ], #8
  74 # LDR     input_high, [ input_ptr  ], #8
  75 # EOR     input_low, k14_low
  76 # EOR     input_high, k14_high
  77 # INS     res_curr.d[0], input_low
  78 # INS     res_curr.d[1], input_high
  79 # AESE    ctr_curr, k0; AESMC ctr_curr, ctr_curr
  80 # AESE    ctr_curr, k1; AESMC ctr_curr, ctr_curr
  81 # AESE    ctr_curr, k2; AESMC ctr_curr, ctr_curr
  82 # AESE    ctr_curr, k3; AESMC ctr_curr, ctr_curr
  83 # AESE    ctr_curr, k4; AESMC ctr_curr, ctr_curr
  84 # AESE    ctr_curr, k5; AESMC ctr_curr, ctr_curr
  85 # AESE    ctr_curr, k6; AESMC ctr_curr, ctr_curr
  86 # AESE    ctr_curr, k7; AESMC ctr_curr, ctr_curr
  87 # AESE    ctr_curr, k8; AESMC ctr_curr, ctr_curr
  88 # AESE    ctr_curr, k9; AESMC ctr_curr, ctr_curr
  89 # AESE    ctr_curr, k10; AESMC ctr_curr, ctr_curr
  90 # AESE    ctr_curr, k11; AESMC ctr_curr, ctr_curr
  91 # AESE    ctr_curr, k12; AESMC ctr_curr, ctr_curr
  92 # AESE    ctr_curr, k13
  93 # EOR     res_curr, res_curr, ctr_curr
  94 # ST1     { res_curr.16b  }, [ output_ptr  ], #16
  95 #
  96 #     Decrypt:
  97 # AESE    ctr_curr, k0; AESMC ctr_curr, ctr_curr
  98 # AESE    ctr_curr, k1; AESMC ctr_curr, ctr_curr
  99 # AESE    ctr_curr, k2; AESMC ctr_curr, ctr_curr
 100 # AESE    ctr_curr, k3; AESMC ctr_curr, ctr_curr
 101 # AESE    ctr_curr, k4; AESMC ctr_curr, ctr_curr
 102 # AESE    ctr_curr, k5; AESMC ctr_curr, ctr_curr
 103 # AESE    ctr_curr, k6; AESMC ctr_curr, ctr_curr
 104 # AESE    ctr_curr, k7; AESMC ctr_curr, ctr_curr
 105 # AESE    ctr_curr, k8; AESMC ctr_curr, ctr_curr
 106 # AESE    ctr_curr, k9; AESMC ctr_curr, ctr_curr
 107 # AESE    ctr_curr, k10; AESMC ctr_curr, ctr_curr
 108 # AESE    ctr_curr, k11; AESMC ctr_curr, ctr_curr
 109 # AESE    ctr_curr, k12; AESMC ctr_curr, ctr_curr
 110 # AESE    ctr_curr, k13
 111 # LDR     res_curr, [ input_ptr  ], #16
 112 # EOR     res_curr, res_curr, ctr_curr
 113 # MOV     output_low, res_curr.d[0]
 114 # MOV     output_high, res_curr.d[1]
 115 # EOR     output_low, k14_low
 116 # EOR     output_high, k14_high
 117 # STP     output_low, output_high, [ output_ptr  ], #16
 118
 119 # GHASH block X:
 120 #     Do 128b karatsuba polynomial multiplication on block
 121 #     We only have 64b->128b polynomial multipliers, naively that means we need to do 4 64b multiplies to generate a 128b
 122 #
 123 # multiplication:
 124 #     Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^ (Pmull(Ah,Bl) ^ Pmull(Al,Bh))<<64
 125 #
 126 #     The idea behind Karatsuba multiplication is that we can do just 3 64b multiplies:
 127 #     Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^ (Pmull(Ah^Al,Bh^Bl) ^ Pmull(Ah,Bh) ^ Pmull(Al,Bl))<<64
 128 #
 129 #     There is some complication here because the bit order of GHASH's PMULL is reversed compared to elsewhere, so we are
 130 #     multiplying with "twisted" powers of H
 131 #
 132 # Note: We can PMULL directly into the acc_x in first GHASH of the loop
 133 # Note: For scheduling big cores we want to split the processing to happen over two loop iterations - otherwise the critical
 134 #       path latency dominates the performance
 135 #
 136 #       This has a knock on effect on register pressure, so we have to be a bit more clever with our temporary registers
 137 #       than indicated here
 138 # REV64   res_curr, res_curr
 139 # INS     t_m.d[0], res_curr.d[1]
 140 # EOR     t_m.8B, t_m.8B, res_curr.8B
 141 # PMULL2  t_h, res_curr, HX
 142 # PMULL   t_l, res_curr, HX
 143 # PMULL   t_m, t_m, HX_k
 144 # EOR     acc_h, acc_h, t_h
 145 # EOR     acc_l, acc_l, t_l
 146 # EOR     acc_m, acc_m, t_m
 147 #
 148 # MODULO: take the partial accumulators (~representing sum of 256b multiplication results), from GHASH and do modulo reduction on them
 149 #         There is some complication here because the bit order of GHASH's PMULL is reversed compared to elsewhere, so we are doing modulo
 150 #         with a reversed constant
 151 # EOR3    acc_m, acc_m, acc_l, acc_h                     // Finish off karatsuba processing
 152 # PMULL   t_mod, acc_h, mod_constant
 153 # EXT     acc_h, acc_h, acc_h, #8
 154 # EOR3     acc_m, acc_m, t_mod, acc_h
 155 # PMULL   acc_h, acc_m, mod_constant
 156 # EXT     acc_m, acc_m, acc_m, #8
 157 # EOR3    acc_l, acc_l, acc_m, acc_h
 158
 159 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
 160 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
 161
 162 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
 163 ( $xlate="${dir}arm-xlate.pl" and -f $xlate  ) or
 164 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate ) or
 165 die "can't locate arm-xlate.pl";
 166
 167 die "only for 64 bit" if $flavour !~ /64/;
 168
 169 open OUT,"| \"$^X\" $xlate $flavour $output";
 170 *STDOUT=*OUT;
 171
 172 $code=<<___;
 173 #include "arm_arch.h"
 174
 175 #if __ARM_MAX_ARCH__>=8
 176 ___
 177 $code.=".arch   armv8.2-a+crypto\n.text\n";
 178
 179 $input_ptr="x0";  #argument block
 180 $bit_length="x1";
 181 $output_ptr="x2";
 182 $current_tag="x3";
 183 $counter="x16";
 184 $constant_temp="x15";
 185 $modulo_constant="x10";
 186 $cc="x8";
 187 {
 188 my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7));
 189 my ($temp2_x,$temp3_x)=map("x$_",(13..14));
 190 my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15));
 191 my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15));
 192 my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7));
 193 my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7));
 194 my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15));
 195
 196 my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15));
 197 my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15));
 198 my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15));
 199
 200 my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19));
 201 my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19));
 202
 203 my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25));
 204 my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25));
 205 my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25));
 206 my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25));
 207
 208 my $t0="v16";
 209 my $t0d="d16";
 210
 211 my $t1="v29";
 212 my $t2=$res1;
 213 my $t3=$t1;
 214
 215 my $t4=$res0;
 216 my $t5=$res2;
 217 my $t6=$t0;
 218
 219 my $t7=$res3;
 220 my $t8=$res4;
 221 my $t9=$res5;
 222
 223 my $t10=$res6;
 224 my $t11="v21";
 225 my $t12=$t1;
 226
 227 my $rtmp_ctr="v30";
 228 my $rtmp_ctrq="q30";
 229 my $rctr_inc="v31";
 230 my $rctr_incd="d31";
 231
 232 my $mod_constantd=$t0d;
 233 my $mod_constant=$t0;
 234
 235 my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28));
 236 my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28));
 237 my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28));
 238 my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28));
 239 my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28));
 240 my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28));
 241 my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28));
 242 my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28));
 243 my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28));
 244 my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28));
 245 my $rk2q1="v28.1q";
 246 my $rk3q1="v26.1q";
 247 my $rk4v="v27";
 248
 249
 250 #########################################################################################
 251 # size_t unroll8_eor3_aes_gcm_enc_128_kernel(const unsigned char *in,
 252 #                               size_t len,
 253 #                               unsigned char *out,
 254 #                               const void *key,
 255 #                               unsigned char ivec[16],
 256 #                               u64 *Xi);
 257 #
 258 $code.=<<___;
 259 .global unroll8_eor3_aes_gcm_enc_128_kernel
 260 .type   unroll8_eor3_aes_gcm_enc_128_kernel,%function
 261 .align  4
 262 unroll8_eor3_aes_gcm_enc_128_kernel:
 263         AARCH64_VALID_CALL_TARGET
 264         cbz     x1, .L128_enc_ret
 265         stp     d8, d9, [sp, #-80]!
 266         mov     $counter, x4
 267         mov     $cc, x5
 268         stp     d10, d11, [sp, #16]
 269         stp     d12, d13, [sp, #32]
 270         stp     d14, d15, [sp, #48]
 271         mov     x5, #0xc200000000000000
 272         stp     x5, xzr, [sp, #64]
 273         add     $modulo_constant, sp, #64
 274
 275         mov     $constant_temp, #0x100000000                            @ set up counter increment
 276         movi    $rctr_inc.16b, #0x0
 277         mov     $rctr_inc.d[1], $constant_temp
 278         lsr     $main_end_input_ptr, $bit_length, #3                    @ byte_len
 279         ld1     { $ctr0b}, [$counter]                                   @ CTR block 0
 280
 281         sub     $main_end_input_ptr, $main_end_input_ptr, #1            @ byte_len - 1
 282
 283         and     $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80           @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
 284
 285         rev32   $rtmp_ctr.16b, $ctr0.16b                                @ set up reversed counter
 286
 287         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 0
 288
 289         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 1
 290         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 1
 291
 292         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 2
 293         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 2
 294
 295         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 3
 296         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 3
 297
 298         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 4
 299         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 4
 300
 301         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 5
 302         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 5
 303         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
 304
 305         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 6
 306         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 6
 307
 308         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 7
 309         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 7
 310
 311         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 0
 312         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 0
 313         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 0
 314
 315         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 0
 316         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 0
 317         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 0
 318
 319         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 0
 320         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 0
 321         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
 322
 323         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 1
 324
 325         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 1
 326         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 1
 327         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 1
 328
 329         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 1
 330         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 1
 331         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 1
 332
 333         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 2
 334         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 1
 335         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 2
 336
 337         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 2
 338         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 2
 339         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 2
 340
 341         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 2
 342         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 2
 343         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 2
 344
 345         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 3
 346
 347         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
 348         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 3
 349         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 3
 350
 351         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 3
 352         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 3
 353         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 3
 354
 355         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 3
 356
 357         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 4
 358         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 3
 359         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 4
 360
 361         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 4
 362         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 4
 363         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 4
 364
 365         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 4
 366         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 4
 367         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 4
 368
 369         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 5
 370         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 5
 371         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
 372
 373         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 5
 374         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 5
 375         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 5
 376
 377         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 5
 378         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 5
 379         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 5
 380
 381         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 6
 382         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 6
 383         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 6
 384
 385         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 6
 386         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 6
 387         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 6
 388
 389         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 6
 390         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 6
 391         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
 392
 393         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 7
 394
 395         ld1     { $acc_lb}, [$current_tag]
 396         ext     $acc_lb, $acc_lb, $acc_lb, #8
 397         rev64   $acc_lb, $acc_lb
 398
 399         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 7
 400
 401         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 7
 402         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 7
 403         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 7
 404
 405         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 7
 406         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 7
 407         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 7
 408
 409         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
 410         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
 411         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
 412
 413         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
 414         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
 415         ldr     $rk10q, [$cc, #160]                                     @ load rk10
 416
 417         aese    $ctr3b, $rk9                                            @ AES block 8k+11 - round 9
 418         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
 419         aese    $ctr2b, $rk9                                            @ AES block 8k+10 - round 9
 420
 421         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
 422         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
 423         aese    $ctr6b, $rk9                                            @ AES block 8k+14 - round 9
 424
 425         aese    $ctr4b, $rk9                                            @ AES block 8k+12 - round 9
 426         add     $main_end_input_ptr, $main_end_input_ptr, $input_ptr
 427         aese    $ctr0b, $rk9                                            @ AES block 8k+8 - round 9
 428
 429         aese    $ctr7b, $rk9                                            @ AES block 8k+15 - round 9
 430         aese    $ctr5b, $rk9                                            @ AES block 8k+13 - round 9
 431         aese    $ctr1b, $rk9                                            @ AES block 8k+9 - round 9
 432
 433         add     $end_input_ptr, $input_ptr, $bit_length, lsr #3         @ end_input_ptr
 434         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
 435         b.ge    .L128_enc_tail                                          @ handle tail
 436
 437         ldp     $ctr_t0q, $ctr_t1q, [$input_ptr], #32                   @ AES block 0, 1 - load plaintext
 438
 439         ldp     $ctr_t2q, $ctr_t3q, [$input_ptr], #32                   @ AES block 2, 3 - load plaintext
 440
 441         ldp     $ctr_t4q, $ctr_t5q, [$input_ptr], #32                   @ AES block 4, 5 - load plaintext
 442
 443         ldp     $ctr_t6q, $ctr_t7q, [$input_ptr], #32                   @ AES block 6, 7 - load plaintext
 444         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
 445
 446         eor3    $res0b, $ctr_t0b, $ctr0b, $rk10                         @ AES block 0 - result
 447         rev32   $ctr0.16b, $rtmp_ctr.16b                                @ CTR block 8
 448         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8
 449
 450         eor3    $res1b, $ctr_t1b, $ctr1b, $rk10                         @ AES block 1 - result
 451         stp     $res0q, $res1q, [$output_ptr], #32                      @ AES block 0, 1 - store result
 452
 453         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 9
 454         eor3    $res5b, $ctr_t5b, $ctr5b, $rk10                         @ AES block 5 - result
 455         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 9
 456
 457         eor3    $res2b, $ctr_t2b, $ctr2b, $rk10                         @ AES block 2 - result
 458         eor3    $res6b, $ctr_t6b, $ctr6b, $rk10                         @ AES block 6 - result
 459         eor3    $res4b, $ctr_t4b, $ctr4b, $rk10                         @ AES block 4 - result
 460
 461         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 10
 462         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 10
 463
 464         eor3    $res3b, $ctr_t3b, $ctr3b, $rk10                         @ AES block 3 - result
 465         eor3    $res7b, $ctr_t7b, $ctr7b,$rk10                          @ AES block 7 - result
 466         stp     $res2q, $res3q, [$output_ptr], #32                      @ AES block 2, 3 - store result
 467
 468         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 11
 469         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 11
 470         stp     $res4q, $res5q, [$output_ptr], #32                      @ AES block 4, 5 - store result
 471
 472         stp     $res6q, $res7q, [$output_ptr], #32                      @ AES block 6, 7 - store result
 473
 474         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 12
 475         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 12
 476         b.ge    .L128_enc_prepretail                                    @ do prepretail
 477
 478 .L128_enc_main_loop:                                                    @ main loop start
 479         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
 480         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
 481         ext     $h5.16b, $h5.16b, $h5.16b, #8
 482         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
 483         ext     $h6.16b, $h6.16b, $h6.16b, #8
 484         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
 485
 486         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
 487         rev64   $res0b, $res0b                                          @ GHASH block 8k
 488         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
 489         ext     $h7.16b, $h7.16b, $h7.16b, #8
 490         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
 491         ext     $h8.16b, $h8.16b, $h8.16b, #8
 492
 493         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
 494         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
 495         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
 496
 497         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
 498         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
 499         rev64   $res5b, $res5b                                          @ GHASH block 8k+5 (t0, t1, t2 and t3 free)
 500         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
 501
 502         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
 503         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
 504         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
 505
 506         rev64   $res7b, $res7b                                          @ GHASH block 8k+7 (t0, t1, t2 and t3 free)
 507
 508         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
 509         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
 510         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
 511
 512         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
 513         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
 514         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
 515
 516         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
 517         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
 518         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
 519
 520         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
 521         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
 522         ext     $h3.16b, $h3.16b, $h3.16b, #8
 523         ldr     $h4q, [$current_tag, #112]                              @ load h3l | h3h
 524         ext     $h4.16b, $h4.16b, $h4.16b, #8
 525         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
 526
 527         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
 528         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
 529         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
 530
 531         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
 532         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
 533         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
 534
 535         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
 536         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
 537         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
 538
 539         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
 540         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
 541         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
 542
 543         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
 544         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
 545         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
 546
 547         eor3    $acc_hb, $acc_hb, $t1.16b,$t2.16b                       @ GHASH block 8k+2, 8k+3 - high
 548         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
 549         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
 550
 551         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
 552         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
 553         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
 554
 555         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
 556         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
 557         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
 558
 559         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k        - mid
 560         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
 561         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
 562
 563         rev64   $res6b, $res6b                                          @ GHASH block 8k+6 (t0, t1, and t2 free)
 564         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
 565
 566         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
 567         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
 568         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
 569
 570         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
 571         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
 572         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
 573
 574         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
 575         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
 576         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
 577
 578         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
 579         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
 580         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
 581
 582         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
 583         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
 584         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
 585         rev64   $res4b, $res4b                                          @ GHASH block 8k+4 (t0, t1, and t2 free)
 586
 587         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
 588         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
 589         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
 590
 591         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
 592         ext     $h1.16b, $h1.16b, $h1.16b, #8
 593         ldr     $h2q, [$current_tag, #64]                               @ load h1l | h1h
 594         ext     $h2.16b, $h2.16b, $h2.16b, #8
 595         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
 596         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
 597
 598         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
 599         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
 600
 601         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
 602         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
 603
 604         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
 605         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
 606
 607         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
 608         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
 609
 610         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
 611         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
 612         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
 613
 614         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
 615         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
 616         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
 617
 618         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
 619         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
 620         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
 621
 622         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
 623         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
 624         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
 625
 626         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
 627         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
 628         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
 629
 630         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
 631         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
 632         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
 633
 634         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
 635         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
 636         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
 637
 638         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
 639         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
 640
 641         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
 642         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
 643         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
 644
 645         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
 646         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
 647         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
 648
 649         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
 650         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
 651         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
 652
 653         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
 654         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
 655
 656         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
 657         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
 658         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
 659
 660         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
 661         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
 662         ldp     $ctr_t0q, $ctr_t1q, [$input_ptr], #32                   @ AES block 8k+8, 8k+9 - load plaintext
 663
 664         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
 665         rev32   $h1.16b, $rtmp_ctr.16b                                  @ CTR block 8k+16
 666         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+16
 667
 668         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
 669         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
 670         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
 671
 672         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
 673         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
 674         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
 675
 676         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
 677         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
 678         ldp     $ctr_t2q, $ctr_t3q, [$input_ptr], #32                   @ AES block 8k+10, 8k+11 - load plaintext
 679
 680         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
 681         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
 682         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
 683
 684         pmull   $t11.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
 685         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
 686         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
 687
 688         rev32   $h2.16b, $rtmp_ctr.16b                                  @ CTR block 8k+17
 689         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
 690
 691         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
 692         ldp     $ctr_t4q, $ctr_t5q, [$input_ptr], #32                   @ AES block 8k+12, 8k+13 - load plaintext
 693         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+17
 694
 695         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
 696         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
 697         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
 698
 699         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
 700         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
 701         ldr     $rk10q, [$cc, #160]                                     @ load rk10
 702
 703         ext     $t12.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
 704         rev32   $h3.16b, $rtmp_ctr.16b                                  @ CTR block 8k+18
 705         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+18
 706         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
 707
 708         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
 709         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
 710         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
 711
 712         aese    $ctr2b, $rk9                                            @ AES block 8k+10 - round 9
 713         aese    $ctr4b, $rk9                                            @ AES block 8k+12 - round 9
 714         aese    $ctr1b, $rk9                                            @ AES block 8k+9 - round 9
 715
 716         ldp     $ctr_t6q, $ctr_t7q, [$input_ptr], #32                   @ AES block 8k+14, 8k+15 - load plaintext
 717         rev32   $h4.16b, $rtmp_ctr.16b                                  @ CTR block 8k+19
 718         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+19
 719
 720         cmp     $input_ptr, $main_end_input_ptr                         @ LOOP CONTROL
 721         eor3    $res4b, $ctr_t4b, $ctr4b, $rk10                         @ AES block 4 - result
 722         aese    $ctr7b, $rk9                                            @ AES block 8k+15 - round 9
 723
 724         aese    $ctr6b, $rk9                                            @ AES block 8k+14 - round 9
 725         aese    $ctr3b, $rk9                                            @ AES block 8k+11 - round 9
 726
 727         eor3    $res2b, $ctr_t2b, $ctr2b, $rk10                         @ AES block 8k+10 - result
 728
 729         mov     $ctr2.16b, $h3.16b                                      @ CTR block 8k+18
 730         aese    $ctr0b, $rk9                                            @ AES block 8k+8 - round 9
 731
 732         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 8k+20
 733         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+20
 734
 735         eor3    $res7b, $ctr_t7b, $ctr7b, $rk10                         @ AES block 7 - result
 736         aese    $ctr5b, $rk9                                            @ AES block 8k+13 - round 9
 737         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
 738
 739         eor3    $res1b, $ctr_t1b, $ctr1b, $rk10                         @ AES block 8k+9 - result
 740         eor3    $res3b, $ctr_t3b, $ctr3b, $rk10                         @ AES block 8k+11 - result
 741         mov     $ctr3.16b, $h4.16b                                      @ CTR block 8k+19
 742
 743         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
 744         eor3    $res5b, $ctr_t5b, $ctr5b, $rk10                         @ AES block 5 - result
 745         mov     $ctr1.16b, $h2.16b                                      @ CTR block 8k+17
 746
 747         eor3    $res0b, $ctr_t0b, $ctr0b, $rk10                         @ AES block 8k+8 - result
 748         mov     $ctr0.16b, $h1.16b                                      @ CTR block 8k+16
 749         stp     $res0q, $res1q, [$output_ptr], #32                      @ AES block 8k+8, 8k+9 - store result
 750
 751         stp     $res2q, $res3q, [$output_ptr], #32                      @ AES block 8k+10, 8k+11 - store result
 752         eor3    $res6b, $ctr_t6b, $ctr6b, $rk10                         @ AES block 6 - result
 753
 754         stp     $res4q, $res5q, [$output_ptr], #32                      @ AES block 8k+12, 8k+13 - store result
 755         eor3    $acc_lb, $acc_lb, $acc_hb, $t11.16b                     @ MODULO - fold into low
 756
 757         stp     $res6q, $res7q, [$output_ptr], #32                      @ AES block 8k+14, 8k+15 - store result
 758         b.lt    .L128_enc_main_loop
 759
 760 .L128_enc_prepretail:                                                   @ PREPRETAIL
 761         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
 762         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
 763         ext     $h7.16b, $h7.16b, $h7.16b, #8
 764         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
 765         ext     $h8.16b, $h8.16b, $h8.16b, #8
 766         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
 767
 768         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
 769         ext     $h5.16b, $h5.16b, $h5.16b, #8
 770         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
 771         ext     $h6.16b, $h6.16b, $h6.16b, #8
 772         rev64   $res0b, $res0b                                          @ GHASH block 8k
 773         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
 774
 775         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
 776         ldr     $h78kq, [$current_tag, #192]                            @ load h6k | h5k
 777         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
 778         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
 779
 780         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
 781         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
 782
 783         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
 784
 785         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
 786         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
 787         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
 788
 789         rev64   $res5b, $res5b                                          @ GHASH block 8k+5 (t0, t1, t2 and t3 free)
 790         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
 791
 792         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
 793         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
 794         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
 795
 796         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
 797         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
 798
 799         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
 800         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
 801
 802         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k        - mid
 803         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
 804
 805         rev64   $res4b, $res4b                                          @ GHASH block 8k+4 (t0, t1, and t2 free)
 806         rev64   $res7b, $res7b                                          @ GHASH block 8k+7 (t0, t1, t2 and t3 free)
 807
 808         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
 809
 810         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
 811
 812         rev64   $res6b, $res6b                                          @ GHASH block 8k+6 (t0, t1, and t2 free)
 813
 814         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
 815
 816         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
 817         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
 818
 819         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
 820         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
 821
 822         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
 823         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
 824
 825         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
 826         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
 827         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
 828
 829         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
 830         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
 831
 832         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
 833         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
 834         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
 835
 836         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
 837         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
 838
 839         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
 840         ext     $h3.16b, $h3.16b, $h3.16b, #8
 841         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
 842         ext     $h4.16b, $h4.16b, $h4.16b, #8
 843
 844         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
 845         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
 846         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
 847
 848         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
 849         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
 850
 851         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
 852         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
 853
 854         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
 855         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
 856         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
 857         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
 858
 859         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
 860         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
 861
 862         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
 863         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
 864         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
 865
 866         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
 867         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
 868         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
 869
 870         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
 871         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
 872
 873         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
 874         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
 875         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
 876
 877         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
 878         ext     $h1.16b, $h1.16b, $h1.16b, #8
 879         ldr     $h2q, [$current_tag, #64]                               @ load h1l | h1h
 880         ext     $h2.16b, $h2.16b, $h2.16b, #8
 881         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
 882         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
 883
 884         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
 885         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
 886         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
 887
 888         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
 889         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
 890         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
 891
 892         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
 893         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
 894
 895         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
 896         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
 897         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
 898
 899         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
 900         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
 901         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
 902
 903         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
 904         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
 905         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
 906
 907         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
 908         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
 909         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
 910
 911         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
 912         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
 913         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
 914
 915         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
 916         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
 917         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
 918
 919         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
 920         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
 921
 922         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
 923         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
 924         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
 925
 926         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
 927         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
 928         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
 929
 930         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
 931         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
 932         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
 933
 934         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
 935         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
 936         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
 937
 938         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
 939         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
 940
 941         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
 942         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
 943         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
 944
 945         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
 946         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
 947
 948         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
 949         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
 950         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
 951
 952         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
 953         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
 954         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
 955
 956         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
 957         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
 958         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
 959
 960         pmull   $t11.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
 961         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
 962         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
 963
 964         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
 965         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
 966         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
 967         ext     $t12.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
 968
 969         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
 970         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
 971         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
 972
 973         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
 974         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
 975
 976         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
 977         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
 978
 979         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
 980         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
 981         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
 982         ext     $acc_mb, $acc_mb, $acc_mb, #8                           @ MODULO - other mid alignment
 983
 984         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
 985         eor3    $acc_lb, $acc_lb, $acc_hb, $acc_mb                      @ MODULO - fold into low
 986         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
 987
 988         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
 989         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
 990         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
 991
 992         ldr     $rk10q, [$cc, #160]                                     @ load rk10
 993         aese    $ctr6b, $rk9                                            @ AES block 8k+14 - round 9
 994         aese    $ctr2b, $rk9                                            @ AES block 8k+10 - round 9
 995
 996         aese    $ctr0b, $rk9                                            @ AES block 8k+8 - round 9
 997         aese    $ctr1b, $rk9                                            @ AES block 8k+9 - round 9
 998
 999         aese    $ctr3b, $rk9                                            @ AES block 8k+11 - round 9
1000         aese    $ctr5b, $rk9                                            @ AES block 8k+13 - round 9
1001
1002         aese    $ctr4b, $rk9                                            @ AES block 8k+12 - round 9
1003         aese    $ctr7b, $rk9                                            @ AES block 8k+15 - round 9
1004 .L128_enc_tail:                                                         @ TAIL
1005
1006         sub     $main_end_input_ptr, $end_input_ptr, $input_ptr         @ main_end_input_ptr is number of bytes left to process
1007         ldr     $ctr_t0q, [$input_ptr], #16                             @ AES block 8k+8 - load plaintext
1008
1009         mov     $t1.16b, $rk10
1010         ldp     $h5q, $h56kq, [$current_tag, #128]                      @ load h5l | h5h
1011         ext     $h5.16b, $h5.16b, $h5.16b, #8
1012
1013         eor3    $res1b, $ctr_t0b, $ctr0b, $t1.16b                       @ AES block 8k+8 - result
1014         ext     $t0.16b, $acc_lb, $acc_lb, #8                           @ prepare final partial tag
1015         ldp     $h6q, $h7q, [$current_tag, #160]                        @ load h6l | h6h
1016         ext     $h6.16b, $h6.16b, $h6.16b, #8
1017         ext     $h7.16b, $h7.16b, $h7.16b, #8
1018
1019         ldp     $h78kq, $h8q, [$current_tag, #192]                      @ load h8k | h7k
1020         ext     $h8.16b, $h8.16b, $h8.16b, #8
1021         cmp     $main_end_input_ptr, #112
1022         b.gt    .L128_enc_blocks_more_than_7
1023
1024         mov     $ctr7b, $ctr6b
1025         mov     $ctr6b, $ctr5b
1026         movi    $acc_h.8b, #0
1027
1028         cmp     $main_end_input_ptr, #96
1029         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
1030         mov     $ctr5b, $ctr4b
1031
1032         mov     $ctr4b, $ctr3b
1033         mov     $ctr3b, $ctr2b
1034         mov     $ctr2b, $ctr1b
1035
1036         movi    $acc_l.8b, #0
1037         movi    $acc_m.8b, #0
1038         b.gt    .L128_enc_blocks_more_than_6
1039
1040         mov     $ctr7b, $ctr6b
1041         cmp     $main_end_input_ptr, #80
1042
1043         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
1044         mov     $ctr6b, $ctr5b
1045         mov     $ctr5b, $ctr4b
1046
1047         mov     $ctr4b, $ctr3b
1048         mov     $ctr3b, $ctr1b
1049         b.gt    .L128_enc_blocks_more_than_5
1050
1051         cmp     $main_end_input_ptr, #64
1052         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
1053
1054         mov     $ctr7b, $ctr6b
1055         mov     $ctr6b, $ctr5b
1056
1057         mov     $ctr5b, $ctr4b
1058         mov     $ctr4b, $ctr1b
1059         b.gt    .L128_enc_blocks_more_than_4
1060
1061         mov     $ctr7b, $ctr6b
1062         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
1063         mov     $ctr6b, $ctr5b
1064
1065         mov     $ctr5b, $ctr1b
1066         cmp     $main_end_input_ptr, #48
1067         b.gt    .L128_enc_blocks_more_than_3
1068
1069         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
1070         mov     $ctr7b, $ctr6b
1071         mov     $ctr6b, $ctr1b
1072
1073         cmp     $main_end_input_ptr, #32
1074         ldr     $h34kq, [$current_tag, #96]                                     @ load h4k | h3k
1075         b.gt    .L128_enc_blocks_more_than_2
1076
1077         cmp     $main_end_input_ptr, #16
1078
1079         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
1080         mov     $ctr7b, $ctr1b
1081         b.gt    .L128_enc_blocks_more_than_1
1082
1083         ldr     $h12kq, [$current_tag, #48]                                     @ load h2k | h1k
1084         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
1085         b        .L128_enc_blocks_less_than_1
1086 .L128_enc_blocks_more_than_7:                                           @ blocks left >  7
1087         st1     { $res1b}, [$output_ptr], #16                           @ AES final-7 block  - store result
1088
1089         rev64   $res0b, $res1b                                          @ GHASH final-7 block
1090         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-6 block - load plaintext
1091
1092         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
1093
1094         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-7 block - mid
1095
1096         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH final-7 block - high
1097
1098         ins     $acc_m.d[0], $h78k.d[1]                                 @ GHASH final-7 block - mid
1099
1100         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-7 block - mid
1101         movi    $t0.8b, #0                                              @ supress further partial tag feed in
1102
1103         eor3    $res1b, $ctr_t1b, $ctr1b, $t1.16b                       @ AES final-6 block - result
1104
1105         pmull   $acc_m.1q, $rk4v.1d, $acc_m.1d                          @ GHASH final-7 block - mid
1106         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH final-7 block - low
1107 .L128_enc_blocks_more_than_6:                                           @ blocks left >  6
1108
1109         st1     { $res1b}, [$output_ptr], #16                           @ AES final-6 block - store result
1110
1111         rev64   $res0b, $res1b                                          @ GHASH final-6 block
1112         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-5 block - load plaintext
1113
1114         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
1115
1116         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-6 block - mid
1117
1118         eor3    $res1b, $ctr_t1b, $ctr2b, $t1.16b                       @ AES final-5 block - result
1119         pmull   $rk3q1, $res0.1d, $h7.1d                                @ GHASH final-6 block - low
1120
1121         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-6 block - mid
1122         movi    $t0.8b, #0                                              @ supress further partial tag feed in
1123
1124         pmull   $rk4v.1q, $rk4v.1d, $h78k.1d                            @ GHASH final-6 block - mid
1125         pmull2  $rk2q1, $res0.2d, $h7.2d                                @ GHASH final-6 block - high
1126
1127         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-6 block - low
1128
1129         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-6 block - mid
1130         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-6 block - high
1131 .L128_enc_blocks_more_than_5:                                           @ blocks left >  5
1132
1133         st1     { $res1b}, [$output_ptr], #16                           @ AES final-5 block - store result
1134
1135         rev64   $res0b, $res1b                                          @ GHASH final-5 block
1136
1137         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
1138
1139         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-5 block - mid
1140         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-4 block - load plaintext
1141         pmull2  $rk2q1, $res0.2d, $h6.2d                                @ GHASH final-5 block - high
1142
1143         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-5 block - high
1144
1145         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-5 block - mid
1146
1147         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-5 block - mid
1148
1149         eor3    $res1b, $ctr_t1b, $ctr3b, $t1.16b                       @ AES final-4 block - result
1150         pmull   $rk3q1, $res0.1d, $h6.1d                                @ GHASH final-5 block - low
1151         movi    $t0.8b, #0                                              @ supress further partial tag feed in
1152
1153         pmull2  $rk4v.1q, $rk4v.2d, $h56k.2d                            @ GHASH final-5 block - mid
1154         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-5 block - low
1155
1156         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-5 block - mid
1157 .L128_enc_blocks_more_than_4:                                           @ blocks left >  4
1158
1159         st1     { $res1b}, [$output_ptr], #16                           @ AES final-4 block - store result
1160
1161         rev64   $res0b, $res1b                                          @ GHASH final-4 block
1162
1163         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-3 block - load plaintext
1164
1165         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
1166
1167         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-4 block - mid
1168         movi    $t0.8b, #0                                              @ supress further partial tag feed in
1169         pmull2  $rk2q1, $res0.2d, $h5.2d                                @ GHASH final-4 block - high
1170
1171         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-4 block - mid
1172
1173         pmull   $rk3q1, $res0.1d, $h5.1d                                @ GHASH final-4 block - low
1174
1175         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-4 block - high
1176         pmull   $rk4v.1q, $rk4v.1d, $h56k.1d                            @ GHASH final-4 block - mid
1177
1178         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-4 block - low
1179
1180         eor3    $res1b, $ctr_t1b, $ctr4b, $t1.16b                       @ AES final-3 block - result
1181         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-4 block - mid
1182 .L128_enc_blocks_more_than_3:                                           @ blocks left >  3
1183
1184         st1     { $res1b}, [$output_ptr], #16                           @ AES final-3 block - store result
1185
1186         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
1187         ext     $h4.16b, $h4.16b, $h4.16b, #8
1188
1189         rev64   $res0b, $res1b                                          @ GHASH final-3 block
1190
1191         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
1192         movi    $t0.8b, #0                                              @ supress further partial tag feed in
1193
1194         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-3 block - mid
1195         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
1196         pmull   $rk3q1, $res0.1d, $h4.1d                                @ GHASH final-3 block - low
1197
1198         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-2 block - load plaintext
1199
1200         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-3 block - mid
1201
1202         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-3 block - mid
1203         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-3 block - low
1204
1205         eor3    $res1b, $ctr_t1b, $ctr5b, $t1.16b                       @ AES final-2 block - result
1206
1207         pmull2  $rk4v.1q, $rk4v.2d, $h34k.2d                            @ GHASH final-3 block - mid
1208         pmull2  $rk2q1, $res0.2d, $h4.2d                                @ GHASH final-3 block - high
1209
1210         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-3 block - mid
1211         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-3 block - high
1212 .L128_enc_blocks_more_than_2:                                           @ blocks left >  2
1213
1214         st1     { $res1b}, [$output_ptr], #16                           @ AES final-2 block - store result
1215
1216         rev64   $res0b, $res1b                                          @ GHASH final-2 block
1217
1218         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
1219
1220         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-1 block - load plaintext
1221
1222         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-2 block - mid
1223         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
1224         ext     $h3.16b, $h3.16b, $h3.16b, #8
1225         movi    $t0.8b, #0                                              @ supress further partial tag feed in
1226
1227         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-2 block - mid
1228         eor3    $res1b, $ctr_t1b, $ctr6b, $t1.16b                       @ AES final-1 block - result
1229
1230         pmull2  $rk2q1, $res0.2d, $h3.2d                                @ GHASH final-2 block - high
1231
1232         pmull   $rk3q1, $res0.1d, $h3.1d                                @ GHASH final-2 block - low
1233         pmull   $rk4v.1q, $rk4v.1d, $h34k.1d                            @ GHASH final-2 block - mid
1234
1235         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-2 block - high
1236
1237         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-2 block - mid
1238         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-2 block - low
1239 .L128_enc_blocks_more_than_1:                                           @ blocks left >  1
1240
1241         st1     { $res1b}, [$output_ptr], #16                           @ AES final-1 block - store result
1242
1243         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
1244         ext     $h2.16b, $h2.16b, $h2.16b, #8
1245         rev64   $res0b, $res1b                                          @ GHASH final-1 block
1246         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final block - load plaintext
1247
1248         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
1249
1250         movi    $t0.8b, #0                                              @ supress further partial tag feed in
1251         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-1 block - mid
1252         eor3    $res1b, $ctr_t1b, $ctr7b, $t1.16b                       @ AES final block - result
1253
1254         pmull2  $rk2q1, $res0.2d, $h2.2d                                @ GHASH final-1 block - high
1255
1256         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-1 block - mid
1257
1258         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
1259
1260         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-1 block - mid
1261
1262         pmull   $rk3q1, $res0.1d, $h2.1d                                @ GHASH final-1 block - low
1263         pmull2  $rk4v.1q, $rk4v.2d, $h12k.2d                            @ GHASH final-1 block - mid
1264
1265         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-1 block - high
1266
1267         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-1 block - mid
1268         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-1 block - low
1269 .L128_enc_blocks_less_than_1:                                           @ blocks left <= 1
1270
1271         rev32   $rtmp_ctr.16b, $rtmp_ctr.16b
1272         str     $rtmp_ctrq, [$counter]                                  @ store the updated counter
1273         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
1274
1275         sub     $bit_length, $bit_length, #128                          @ bit_length -= 128
1276
1277         neg     $bit_length, $bit_length                                @ bit_length = 128 - #bits in input (in range [1,128])
1278
1279         mvn     $temp0_x, xzr                                           @ temp0_x = 0xffffffffffffffff
1280         ld1     { $rk0}, [$output_ptr]                                  @ load existing bytes where the possibly partial last block is to be stored
1281         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
1282
1283         lsr     $temp0_x, $temp0_x, $bit_length                         @ temp0_x is mask for top 64b of last block
1284         mvn     $temp1_x, xzr                                           @ temp1_x = 0xffffffffffffffff
1285         cmp     $bit_length, #64
1286
1287         csel    $temp2_x, $temp1_x, $temp0_x, lt
1288         csel    $temp3_x, $temp0_x, xzr, lt
1289
1290         mov     $ctr0.d[1], $temp3_x
1291         mov     $ctr0.d[0], $temp2_x                                    @ ctr0b is mask for last block
1292
1293         and     $res1b, $res1b, $ctr0b                                  @ possibly partial last block has zeroes in highest bits
1294
1295         rev64   $res0b, $res1b                                          @ GHASH final block
1296
1297         bif     $res1b, $rk0, $ctr0b                                    @ insert existing bytes in top end of result before storing
1298         st1     { $res1b}, [$output_ptr]                                @ store all 16B
1299
1300         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
1301
1302         ins     $t0.d[0], $res0.d[1]                                    @ GHASH final block - mid
1303
1304         eor     $t0.8b, $t0.8b, $res0.8b                                @ GHASH final block - mid
1305         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
1306         ext     $h1.16b, $h1.16b, $h1.16b, #8
1307
1308         pmull   $t0.1q, $t0.1d, $h12k.1d                                @ GHASH final block - mid
1309
1310         pmull2  $rk2q1, $res0.2d, $h1.2d                                @ GHASH final block - high
1311         eor     $acc_mb, $acc_mb, $t0.16b                               @ GHASH final block - mid
1312         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
1313
1314         pmull   $rk3q1, $res0.1d, $h1.1d                                @ GHASH final block - low
1315
1316         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final block - high
1317
1318         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final block - low
1319
1320         ext     $t11.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
1321         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
1322
1323         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
1324
1325         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
1326
1327         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
1328         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
1329
1330         eor3    $acc_lb, $acc_lb, $acc_hb, $t11.16b                     @ MODULO - fold into low
1331         ext     $acc_lb, $acc_lb, $acc_lb, #8
1332         rev64   $acc_lb, $acc_lb
1333         st1     { $acc_l.16b }, [$current_tag]
1334         lsr     x0, $bit_length, #3                                     @ return sizes
1335
1336         ldp     d10, d11, [sp, #16]
1337         ldp     d12, d13, [sp, #32]
1338         ldp     d14, d15, [sp, #48]
1339         ldp     d8, d9, [sp], #80
1340         ret
1341
1342 .L128_enc_ret:
1343         mov w0, #0x0
1344         ret
1345 .size unroll8_eor3_aes_gcm_enc_128_kernel,.-unroll8_eor3_aes_gcm_enc_128_kernel
1346 ___
1347
1348 #########################################################################################
1349 # size_t unroll8_eor3_aes_gcm_dec_128_kernel(const unsigned char *in,
1350 #                               size_t len,
1351 #                               unsigned char *out,
1352 #                               u64 *Xi,
1353 #                               unsigned char ivec[16],
1354 #                               const void *key);
1355 #
1356 $code.=<<___;
1357 .global unroll8_eor3_aes_gcm_dec_128_kernel
1358 .type   unroll8_eor3_aes_gcm_dec_128_kernel,%function
1359 .align  4
1360 unroll8_eor3_aes_gcm_dec_128_kernel:
1361         AARCH64_VALID_CALL_TARGET
1362         cbz     x1, .L128_dec_ret
1363         stp     d8, d9, [sp, #-80]!
1364         mov     $counter, x4
1365         mov     $cc, x5
1366         stp     d10, d11, [sp, #16]
1367         stp     d12, d13, [sp, #32]
1368         stp     d14, d15, [sp, #48]
1369         mov     x5, #0xc200000000000000
1370         stp     x5, xzr, [sp, #64]
1371         add     $modulo_constant, sp, #64
1372
1373         lsr     $main_end_input_ptr, $bit_length, #3                    @ byte_len
1374         ld1     { $ctr0b}, [$counter]                                   @ CTR block 0
1375
1376         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
1377         sub     $main_end_input_ptr, $main_end_input_ptr, #1            @ byte_len - 1
1378
1379         mov     $constant_temp, #0x100000000                            @ set up counter increment
1380         movi    $rctr_inc.16b, #0x0
1381         mov     $rctr_inc.d[1], $constant_temp
1382         ld1     { $acc_lb}, [$current_tag]
1383           ext   $acc_lb, $acc_lb, $acc_lb, #8
1384         rev64   $acc_lb, $acc_lb
1385
1386         rev32   $rtmp_ctr.16b, $ctr0.16b                                @ set up reversed counter
1387
1388         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 0
1389
1390         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 0
1391
1392         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 1
1393         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 1
1394
1395         and     $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80   @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
1396
1397         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 2
1398         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 2
1399         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 0
1400
1401         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 3
1402         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 3
1403
1404         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 1
1405         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 1
1406
1407         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 4
1408         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 4
1409
1410         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 5
1411         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 5
1412
1413         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 0
1414
1415         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 6
1416         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 6
1417         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 0
1418
1419         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 0
1420         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 0
1421
1422         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 7
1423
1424         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 0
1425         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 1
1426
1427         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 0
1428
1429         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
1430
1431         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 1
1432         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 1
1433
1434         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 1
1435         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 1
1436
1437         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 2
1438         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 2
1439         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 1
1440
1441         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 2
1442         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 2
1443         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 2
1444
1445         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 2
1446         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 2
1447         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 2
1448
1449         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 3
1450         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 3
1451
1452         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
1453         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 3
1454
1455         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 3
1456         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 3
1457
1458         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 3
1459         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 3
1460
1461         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 4
1462         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 4
1463         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 3
1464
1465         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 4
1466         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 4
1467         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 4
1468
1469         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 4
1470         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 4
1471         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 4
1472
1473         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
1474         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 5
1475         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 5
1476
1477         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 5
1478         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 5
1479
1480         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 5
1481         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 5
1482
1483         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 5
1484
1485         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 6
1486         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 6
1487         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 5
1488
1489         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 6
1490         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 6
1491         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 6
1492
1493         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 6
1494         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 6
1495         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 6
1496
1497         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 7
1498         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 7
1499         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 7
1500
1501         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 7
1502         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 7
1503         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
1504
1505         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 7
1506         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 7
1507         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 7
1508
1509         add     $main_end_input_ptr, $main_end_input_ptr, $input_ptr
1510         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 7
1511
1512         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 8
1513         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 8
1514
1515         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 8
1516         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 8
1517         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 8
1518
1519         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 8
1520         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 8
1521         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 8
1522
1523         aese    $ctr0b, $rk9                                            @ AES block 0 - round 9
1524         aese    $ctr1b, $rk9                                            @ AES block 1 - round 9
1525         aese    $ctr6b, $rk9                                            @ AES block 6 - round 9
1526
1527         ldr     $rk10q, [$cc, #160]                                     @ load rk10
1528         aese    $ctr4b, $rk9                                            @ AES block 4 - round 9
1529         aese    $ctr3b, $rk9                                            @ AES block 3 - round 9
1530
1531         aese    $ctr2b, $rk9                                            @ AES block 2 - round 9
1532         aese    $ctr5b, $rk9                                            @ AES block 5 - round 9
1533         aese    $ctr7b, $rk9                                            @ AES block 7 - round 9
1534
1535         add     $end_input_ptr, $input_ptr, $bit_length, lsr #3         @ end_input_ptr
1536         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
1537         b.ge    .L128_dec_tail                                          @ handle tail
1538
1539         ldp     $res0q, $res1q, [$input_ptr], #32                       @ AES block 0, 1 - load ciphertext
1540
1541         eor3    $ctr0b, $res0b, $ctr0b, $rk10                           @ AES block 0 - result
1542         eor3    $ctr1b, $res1b, $ctr1b, $rk10                           @ AES block 1 - result
1543         stp     $ctr0q, $ctr1q, [$output_ptr], #32                      @ AES block 0, 1 - store result
1544
1545         rev32   $ctr0.16b, $rtmp_ctr.16b                                @ CTR block 8
1546         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8
1547         ldp     $res2q, $res3q, [$input_ptr], #32                       @ AES block 2, 3 - load ciphertext
1548
1549         ldp     $res4q, $res5q, [$input_ptr], #32                       @ AES block 4, 5 - load ciphertext
1550
1551         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 9
1552         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 9
1553         ldp     $res6q, $res7q, [$input_ptr], #32                       @ AES block 6, 7 - load ciphertext
1554
1555         eor3    $ctr3b, $res3b, $ctr3b, $rk10                           @ AES block 3 - result
1556         eor3    $ctr2b, $res2b, $ctr2b, $rk10                           @ AES block 2 - result
1557         stp     $ctr2q, $ctr3q, [$output_ptr], #32                      @ AES block 2, 3 - store result
1558
1559         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 10
1560         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 10
1561
1562         eor3    $ctr6b, $res6b, $ctr6b, $rk10                           @ AES block 6 - result
1563
1564         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 11
1565         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 11
1566
1567         eor3    $ctr4b, $res4b, $ctr4b, $rk10                           @ AES block 4 - result
1568         eor3    $ctr5b, $res5b, $ctr5b, $rk10                           @ AES block 5 - result
1569         stp     $ctr4q, $ctr5q, [$output_ptr], #32                      @ AES block 4, 5 - store result
1570
1571         eor3    $ctr7b, $res7b, $ctr7b, $rk10                           @ AES block 7 - result
1572         stp     $ctr6q, $ctr7q, [$output_ptr], #32                      @ AES block 6, 7 - store result
1573         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 12
1574
1575         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
1576         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 12
1577         b.ge    .L128_dec_prepretail                                    @ do prepretail
1578
1579 .L128_dec_main_loop:                                                    @ main loop start
1580         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
1581         ext     $h7.16b, $h7.16b, $h7.16b, #8
1582         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
1583         ext     $h8.16b, $h8.16b, $h8.16b, #8
1584
1585         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
1586         rev64   $res0b, $res0b                                          @ GHASH block 8k
1587         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
1588
1589         rev64   $res6b, $res6b                                          @ GHASH block 8k+6
1590         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
1591         ext     $h5.16b, $h5.16b, $h5.16b, #8
1592         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
1593         ext     $h6.16b, $h6.16b, $h6.16b, #8
1594
1595         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
1596         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
1597         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
1598
1599         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
1600         rev64   $res4b, $res4b                                          @ GHASH block 8k+4
1601         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
1602
1603         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
1604         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
1605         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
1606         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
1607
1608         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
1609         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
1610         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
1611
1612         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
1613         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
1614         rev64   $res5b, $res5b                                          @ GHASH block 8k+5
1615
1616         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
1617         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
1618         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
1619
1620         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
1621         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
1622         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
1623
1624         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
1625         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
1626         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
1627
1628         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
1629         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
1630         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
1631
1632         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
1633         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
1634         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
1635
1636         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
1637         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
1638         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
1639
1640         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
1641         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
1642         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
1643
1644         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
1645         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
1646         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k        - mid
1647
1648         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
1649         ext     $h3.16b, $h3.16b, $h3.16b, #8
1650         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
1651         ext     $h4.16b, $h4.16b, $h4.16b, #8
1652         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
1653         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
1654
1655         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
1656         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
1657         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
1658
1659         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
1660         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
1661         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
1662
1663         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
1664         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
1665         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
1666
1667         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
1668         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
1669         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
1670         ext     $h1.16b, $h1.16b, $h1.16b, #8
1671         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
1672         ext     $h2.16b, $h2.16b, $h2.16b, #8
1673
1674         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
1675         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
1676         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
1677
1678         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
1679         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
1680         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
1681
1682         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
1683         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
1684         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
1685
1686         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
1687         rev64   $res7b, $res7b                                          @ GHASH block 8k+7
1688         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
1689
1690         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
1691         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
1692         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
1693
1694         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
1695         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
1696         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
1697         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
1698
1699         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
1700         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
1701         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
1702
1703         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
1704         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
1705         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
1706
1707         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
1708         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
1709         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
1710
1711         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
1712         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
1713         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
1714
1715         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
1716         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
1717         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
1718
1719         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
1720         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
1721         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
1722
1723         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
1724         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
1725         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
1726
1727         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
1728         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
1729         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
1730
1731         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
1732         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
1733         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
1734
1735         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
1736         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
1737         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
1738
1739         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
1740         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
1741         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
1742
1743         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
1744         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
1745         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
1746
1747         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
1748         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
1749         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
1750
1751         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
1752         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
1753         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
1754
1755         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
1756         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
1757         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
1758
1759         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
1760         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
1761         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
1762
1763         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
1764         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
1765         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
1766
1767         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
1768         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
1769         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
1770
1771         rev32   $h1.16b, $rtmp_ctr.16b                                  @ CTR block 8k+16
1772         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
1773         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+16
1774
1775         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
1776         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
1777         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
1778
1779         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
1780         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
1781         rev32   $h2.16b, $rtmp_ctr.16b                                  @ CTR block 8k+17
1782
1783         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
1784         ext     $t11.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
1785         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
1786
1787         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
1788         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
1789         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+17
1790
1791         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
1792         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
1793         ldp     $res0q, $res1q, [$input_ptr], #32                       @ AES block 8k+8, 8k+9 - load ciphertext
1794
1795         ldp     $res2q, $res3q, [$input_ptr], #32                       @ AES block 8k+10, 8k+11 - load ciphertext
1796         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
1797         rev32   $h3.16b, $rtmp_ctr.16b                                  @ CTR block 8k+18
1798
1799         ldp     $res4q, $res5q, [$input_ptr], #32                       @ AES block 8k+12, 8k+13 - load ciphertext
1800         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
1801         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
1802
1803         ldp     $res6q, $res7q, [$input_ptr], #32                       @ AES block 8k+14, 8k+15 - load ciphertext
1804         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
1805         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+18
1806
1807         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
1808         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
1809         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
1810
1811         aese    $ctr0b, $rk9                                            @ AES block 8k+8 - round 9
1812         aese    $ctr1b, $rk9                                            @ AES block 8k+9 - round 9
1813         ldr     $rk10q, [$cc, #160]                                     @ load rk10
1814
1815         aese    $ctr6b, $rk9                                            @ AES block 8k+14 - round 9
1816         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
1817         aese    $ctr2b, $rk9                                            @ AES block 8k+10 - round 9
1818
1819         aese    $ctr7b, $rk9                                            @ AES block 8k+15 - round 9
1820         aese    $ctr4b, $rk9                                            @ AES block 8k+12 - round 9
1821         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
1822
1823         rev32   $h4.16b, $rtmp_ctr.16b                                  @ CTR block 8k+19
1824         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+19
1825
1826         aese    $ctr3b, $rk9                                            @ AES block 8k+11 - round 9
1827         aese    $ctr5b, $rk9                                            @ AES block 8k+13 - round 9
1828         eor3    $ctr1b, $res1b, $ctr1b, $rk10                           @ AES block 8k+9 - result
1829
1830         eor3    $ctr0b, $res0b, $ctr0b, $rk10                           @ AES block 8k+8 - result
1831         eor3    $ctr7b, $res7b, $ctr7b, $rk10                           @ AES block 8k+15 - result
1832         eor3    $ctr6b, $res6b, $ctr6b, $rk10                           @ AES block 8k+14 - result
1833
1834         eor3    $ctr2b, $res2b, $ctr2b, $rk10                           @ AES block 8k+10 - result
1835         stp     $ctr0q, $ctr1q, [$output_ptr], #32                      @ AES block 8k+8, 8k+9 - store result
1836         mov     $ctr1.16b, $h2.16b                                      @ CTR block 8k+17
1837
1838         eor3    $ctr4b, $res4b, $ctr4b, $rk10                           @ AES block 8k+12 - result
1839         eor3    $acc_lb, $acc_lb, $acc_hb, $t11.16b                     @ MODULO - fold into low
1840         mov     $ctr0.16b, $h1.16b                                      @ CTR block 8k+16
1841
1842         eor3    $ctr3b, $res3b, $ctr3b, $rk10                           @ AES block 8k+11 - result
1843         cmp     $input_ptr, $main_end_input_ptr                         @ LOOP CONTROL
1844         stp     $ctr2q, $ctr3q, [$output_ptr], #32                      @ AES block 8k+10, 8k+11 - store result
1845
1846         eor3    $ctr5b, $res5b, $ctr5b, $rk10                           @ AES block 8k+13 - result
1847         mov     $ctr2.16b, $h3.16b                                      @ CTR block 8k+18
1848
1849         stp     $ctr4q, $ctr5q, [$output_ptr], #32                      @ AES block 8k+12, 8k+13 - store result
1850         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 8k+20
1851         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+20
1852
1853         stp     $ctr6q, $ctr7q, [$output_ptr], #32                      @ AES block 8k+14, 8k+15 - store result
1854         mov     $ctr3.16b, $h4.16b                                      @ CTR block 8k+19
1855         b.lt    .L128_dec_main_loop
1856
1857 .L128_dec_prepretail:                                                   @ PREPRETAIL
1858         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
1859         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
1860         rev64   $res0b, $res0b                                          @ GHASH block 8k
1861
1862         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
1863         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
1864         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
1865
1866         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
1867         ext     $h7.16b, $h7.16b, $h7.16b, #8
1868         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
1869         ext     $h8.16b, $h8.16b, $h8.16b, #8
1870         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
1871         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
1872
1873         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
1874         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
1875         ext     $h5.16b, $h5.16b, $h5.16b, #8
1876         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
1877         ext     $h6.16b, $h6.16b, $h6.16b, #8
1878         rev64   $res5b, $res5b                                          @ GHASH block 8k+5
1879
1880         rev64   $res4b, $res4b                                          @ GHASH block 8k+4
1881
1882         rev64   $res6b, $res6b                                          @ GHASH block 8k+6
1883
1884         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
1885         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
1886         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
1887         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
1888
1889         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
1890         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
1891         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
1892
1893         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
1894         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
1895         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
1896
1897         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
1898         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
1899         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
1900
1901         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
1902         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
1903         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
1904
1905         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
1906         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
1907         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
1908
1909         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
1910         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
1911         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
1912
1913         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
1914         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
1915         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
1916
1917         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k - mid
1918         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
1919         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
1920
1921         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
1922         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
1923         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
1924
1925         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
1926         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
1927         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
1928
1929         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
1930         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
1931         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
1932
1933         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
1934         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
1935         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
1936
1937         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
1938         ext     $h3.16b, $h3.16b, $h3.16b, #8
1939         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
1940         ext     $h4.16b, $h4.16b, $h4.16b, #8
1941         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
1942         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
1943
1944         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
1945         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
1946         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
1947
1948         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
1949         ext     $h1.16b, $h1.16b, $h1.16b, #8
1950         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
1951         ext     $h2.16b, $h2.16b, $h2.16b, #8
1952         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
1953
1954         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
1955         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
1956         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
1957
1958         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
1959         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
1960         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
1961
1962         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
1963         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
1964         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
1965
1966         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
1967         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
1968         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
1969
1970         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
1971         rev64   $res7b, $res7b                                          @ GHASH block 8k+7
1972         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
1973
1974         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
1975         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
1976         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
1977         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
1978
1979         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
1980         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
1981         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
1982
1983         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
1984         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
1985         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
1986
1987         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
1988         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
1989         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
1990
1991         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
1992         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
1993         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
1994
1995         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
1996         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
1997         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
1998
1999         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
2000         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
2001         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
2002
2003         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
2004         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
2005         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
2006
2007         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
2008         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
2009         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
2010
2011         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
2012         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
2013         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
2014
2015         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
2016         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
2017         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
2018
2019         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
2020         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
2021         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
2022
2023         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
2024         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
2025         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
2026
2027         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
2028         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
2029         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
2030
2031         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
2032         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
2033         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
2034
2035         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
2036         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
2037         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
2038
2039         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
2040         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
2041         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
2042
2043         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
2044         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
2045         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
2046
2047         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
2048         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
2049         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
2050
2051         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
2052         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
2053         ext     $t11.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
2054
2055         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
2056         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
2057         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
2058
2059         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
2060         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
2061         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
2062
2063         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
2064         ldr     $rk10q, [$cc, #160]                                     @ load rk10
2065
2066         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
2067         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
2068
2069         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
2070         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
2071         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
2072
2073         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
2074         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
2075         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
2076
2077         aese    $ctr6b, $rk9                                            @ AES block 8k+14 - round 9
2078         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
2079         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
2080
2081         eor3    $acc_lb, $acc_lb, $acc_hb, $t11.16b                     @ MODULO - fold into low
2082         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
2083         aese    $ctr2b, $rk9                                            @ AES block 8k+10 - round 9
2084
2085         aese    $ctr3b, $rk9                                            @ AES block 8k+11 - round 9
2086         aese    $ctr5b, $rk9                                            @ AES block 8k+13 - round 9
2087         aese    $ctr0b, $rk9                                            @ AES block 8k+8 - round 9
2088
2089         aese    $ctr4b, $rk9                                            @ AES block 8k+12 - round 9
2090         aese    $ctr1b, $rk9                                            @ AES block 8k+9 - round 9
2091         aese    $ctr7b, $rk9                                            @ AES block 8k+15 - round 9
2092
2093 .L128_dec_tail:                                                         @ TAIL
2094
2095         mov     $t1.16b, $rk10
2096         sub     $main_end_input_ptr, $end_input_ptr, $input_ptr         @ main_end_input_ptr is number of bytes left to process
2097
2098         cmp     $main_end_input_ptr, #112
2099
2100         ldp     $h78kq, $h8q, [$current_tag, #192]                      @ load h8k | h7k
2101         ext     $h8.16b, $h8.16b, $h8.16b, #8
2102         ldr     $res1q, [$input_ptr], #16                               @ AES block 8k+8 - load ciphertext
2103
2104         ldp     $h5q, $h56kq, [$current_tag, #128]                      @ load h5l | h5h
2105         ext     $h5.16b, $h5.16b, $h5.16b, #8
2106         ext     $t0.16b, $acc_lb, $acc_lb, #8                           @ prepare final partial tag
2107
2108         ldp     $h6q, $h7q, [$current_tag, #160]                        @ load h6l | h6h
2109         ext     $h6.16b, $h6.16b, $h6.16b, #8
2110         ext     $h7.16b, $h7.16b, $h7.16b, #8
2111
2112         eor3    $res4b, $res1b, $ctr0b, $t1.16b                         @ AES block 8k+8 - result
2113         b.gt    .L128_dec_blocks_more_than_7
2114
2115         cmp     $main_end_input_ptr, #96
2116         mov     $ctr7b, $ctr6b
2117         movi    $acc_l.8b, #0
2118
2119         movi    $acc_h.8b, #0
2120         mov     $ctr6b, $ctr5b
2121         mov     $ctr5b, $ctr4b
2122
2123         mov     $ctr4b, $ctr3b
2124         mov     $ctr3b, $ctr2b
2125         mov     $ctr2b, $ctr1b
2126
2127         movi    $acc_m.8b, #0
2128         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
2129         b.gt    .L128_dec_blocks_more_than_6
2130
2131         cmp     $main_end_input_ptr, #80
2132         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
2133
2134         mov     $ctr7b, $ctr6b
2135         mov     $ctr6b, $ctr5b
2136         mov     $ctr5b, $ctr4b
2137
2138         mov     $ctr4b, $ctr3b
2139         mov     $ctr3b, $ctr1b
2140         b.gt    .L128_dec_blocks_more_than_5
2141
2142         cmp     $main_end_input_ptr, #64
2143
2144         mov     $ctr7b, $ctr6b
2145         mov     $ctr6b, $ctr5b
2146         mov     $ctr5b, $ctr4b
2147
2148         mov     $ctr4b, $ctr1b
2149         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
2150         b.gt    .L128_dec_blocks_more_than_4
2151
2152         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
2153         mov     $ctr7b, $ctr6b
2154         mov     $ctr6b, $ctr5b
2155
2156         mov     $ctr5b, $ctr1b
2157         cmp     $main_end_input_ptr, #48
2158         b.gt    .L128_dec_blocks_more_than_3
2159
2160         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
2161         mov     $ctr7b, $ctr6b
2162         cmp     $main_end_input_ptr, #32
2163
2164         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
2165         mov     $ctr6b, $ctr1b
2166         b.gt    .L128_dec_blocks_more_than_2
2167
2168         cmp     $main_end_input_ptr, #16
2169
2170         mov     $ctr7b, $ctr1b
2171         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
2172         b.gt    L128_dec_blocks_more_than_1
2173
2174         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
2175         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
2176         b        .L128_dec_blocks_less_than_1
2177 .L128_dec_blocks_more_than_7:                                           @ blocks left >  7
2178         rev64   $res0b, $res1b                                          @ GHASH final-7 block
2179
2180         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
2181
2182         ins     $acc_m.d[0], $h78k.d[1]                                 @ GHASH final-7 block - mid
2183
2184         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH final-7 block - low
2185         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-7 block - mid
2186
2187         movi    $t0.8b, #0                                              @ supress further partial tag feed in
2188         ldr     $res1q, [$input_ptr], #16                               @ AES final-6 block - load ciphertext
2189
2190         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-7 block - mid
2191
2192         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH final-7 block - high
2193         st1     { $res4b}, [$output_ptr], #16                           @ AES final-7 block  - store result
2194         eor3    $res4b, $res1b, $ctr1b, $t1.16b                         @ AES final-6 block - result
2195
2196         pmull   $acc_m.1q, $rk4v.1d, $acc_m.1d                          @ GHASH final-7 block - mid
2197 .L128_dec_blocks_more_than_6:                                           @ blocks left >  6
2198
2199         rev64   $res0b, $res1b                                          @ GHASH final-6 block
2200
2201         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
2202
2203         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-6 block - mid
2204
2205         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-6 block - mid
2206
2207         pmull   $rk3q1, $res0.1d, $h7.1d                                @ GHASH final-6 block - low
2208         ldr     $res1q, [$input_ptr], #16                               @ AES final-5 block - load ciphertext
2209         movi    $t0.8b, #0                                              @ supress further partial tag feed in
2210
2211         pmull   $rk4v.1q, $rk4v.1d, $h78k.1d                            @ GHASH final-6 block - mid
2212         st1     { $res4b}, [$output_ptr], #16                           @ AES final-6 block - store result
2213         pmull2  $rk2q1, $res0.2d, $h7.2d                                @ GHASH final-6 block - high
2214
2215         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-6 block - low
2216         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-6 block - high
2217
2218         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-6 block - mid
2219         eor3    $res4b, $res1b, $ctr2b, $t1.16b                         @ AES final-5 block - result
2220 .L128_dec_blocks_more_than_5:                                           @ blocks left >  5
2221
2222         rev64   $res0b, $res1b                                          @ GHASH final-5 block
2223
2224         ldr     $res1q, [$input_ptr], #16                               @ AES final-4 block - load ciphertext
2225         st1     { $res4b}, [$output_ptr], #16                           @ AES final-5 block - store result
2226
2227         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
2228
2229         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-5 block - mid
2230
2231         eor3    $res4b, $res1b, $ctr3b, $t1.16b                         @ AES final-4 block - result
2232
2233         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-5 block - mid
2234
2235         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-5 block - mid
2236         pmull   $rk3q1, $res0.1d, $h6.1d                                @ GHASH final-5 block - low
2237         movi    $t0.8b, #0                                              @ supress further partial tag feed in
2238
2239         pmull2  $rk4v.1q, $rk4v.2d, $h56k.2d                            @ GHASH final-5 block - mid
2240         pmull2  $rk2q1, $res0.2d, $h6.2d                                @ GHASH final-5 block - high
2241         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-5 block - low
2242
2243         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-5 block - mid
2244         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-5 block - high
2245 .L128_dec_blocks_more_than_4:                                           @ blocks left >  4
2246
2247         rev64   $res0b, $res1b                                          @ GHASH final-4 block
2248
2249         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
2250         ldr     $res1q, [$input_ptr], #16                               @ AES final-3 block - load ciphertext
2251
2252         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-4 block - mid
2253         movi    $t0.8b, #0                                              @ supress further partial tag feed in
2254         pmull2  $rk2q1, $res0.2d, $h5.2d                                @ GHASH final-4 block - high
2255
2256         pmull   $rk3q1, $res0.1d, $h5.1d                                @ GHASH final-4 block - low
2257
2258         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-4 block - high
2259
2260         st1     { $res4b}, [$output_ptr], #16                           @ AES final-4 block - store result
2261         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-4 block - mid
2262
2263         eor3    $res4b, $res1b, $ctr4b, $t1.16b                         @ AES final-3 block - result
2264         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-4 block - low
2265
2266         pmull   $rk4v.1q, $rk4v.1d, $h56k.1d                            @ GHASH final-4 block - mid
2267
2268         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-4 block - mid
2269 .L128_dec_blocks_more_than_3:                                           @ blocks left >  3
2270
2271         st1     { $res4b}, [$output_ptr], #16                           @ AES final-3 block - store result
2272         rev64   $res0b, $res1b                                          @ GHASH final-3 block
2273
2274         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
2275
2276         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-3 block - mid
2277
2278         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
2279         ext     $h4.16b, $h4.16b, $h4.16b, #8
2280         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
2281
2282         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-3 block - mid
2283
2284         ldr     $res1q, [$input_ptr], #16                               @ AES final-2 block - load ciphertext
2285
2286         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-3 block - mid
2287         pmull   $rk3q1, $res0.1d, $h4.1d                                @ GHASH final-3 block - low
2288         pmull2  $rk2q1, $res0.2d, $h4.2d                                @ GHASH final-3 block - high
2289
2290         movi    $t0.8b, #0                                              @ supress further partial tag feed in
2291         eor3    $res4b, $res1b, $ctr5b, $t1.16b                         @ AES final-2 block - result
2292         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-3 block - low
2293
2294         pmull2  $rk4v.1q, $rk4v.2d, $h34k.2d                            @ GHASH final-3 block - mid
2295
2296         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-3 block - high
2297         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-3 block - mid
2298 .L128_dec_blocks_more_than_2:                                           @ blocks left >  2
2299
2300         rev64   $res0b, $res1b                                          @ GHASH final-2 block
2301
2302         st1     { $res4b}, [$output_ptr], #16                           @ AES final-2 block - store result
2303
2304         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
2305         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
2306         ext     $h3.16b, $h3.16b, $h3.16b, #8
2307         movi    $t0.8b, #0                                              @ supress further partial tag feed in
2308
2309         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-2 block - mid
2310
2311         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-2 block - mid
2312
2313         pmull   $rk3q1, $res0.1d, $h3.1d                                @ GHASH final-2 block - low
2314
2315         pmull2  $rk2q1, $res0.2d, $h3.2d                                @ GHASH final-2 block - high
2316         pmull   $rk4v.1q, $rk4v.1d, $h34k.1d                            @ GHASH final-2 block - mid
2317         ldr     $res1q, [$input_ptr], #16                               @ AES final-1 block - load ciphertext
2318
2319         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-2 block - mid
2320
2321         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-2 block - low
2322
2323         eor3    $res4b, $res1b, $ctr6b, $t1.16b                         @ AES final-1 block - result
2324         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-2 block - high
2325 .L128_dec_blocks_more_than_1:                                           @ blocks left >  1
2326
2327         st1     { $res4b}, [$output_ptr], #16                           @ AES final-1 block - store result
2328         rev64   $res0b, $res1b                                          @ GHASH final-1 block
2329
2330         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
2331         ext     $h2.16b, $h2.16b, $h2.16b, #8
2332
2333         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
2334
2335         movi    $t0.8b, #0                                              @ supress further partial tag feed in
2336
2337         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-1 block - mid
2338
2339         ldr     $res1q, [$input_ptr], #16                               @ AES final block - load ciphertext
2340         pmull2  $rk2q1, $res0.2d, $h2.2d                                @ GHASH final-1 block - high
2341
2342         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-1 block - mid
2343         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-1 block - high
2344         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
2345
2346         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-1 block - mid
2347         eor3    $res4b, $res1b, $ctr7b, $t1.16b                         @ AES final block - result
2348
2349         pmull   $rk3q1, $res0.1d, $h2.1d                                @ GHASH final-1 block - low
2350
2351         pmull2  $rk4v.1q, $rk4v.2d, $h12k.2d                            @ GHASH final-1 block - mid
2352
2353         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-1 block - low
2354
2355         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-1 block - mid
2356 .L128_dec_blocks_less_than_1:                                           @ blocks left <= 1
2357
2358         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
2359
2360         sub     $bit_length, $bit_length, #128                          @ bit_length -= 128
2361
2362         neg     $bit_length, $bit_length                                @ bit_length = 128 - #bits in input (in range [1,128])
2363
2364         mvn     $temp0_x, xzr                                           @ temp0_x = 0xffffffffffffffff
2365         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
2366
2367         lsr     $temp0_x, $temp0_x, $bit_length                         @ temp0_x is mask for top 64b of last block
2368         cmp     $bit_length, #64
2369         mvn     $temp1_x, xzr                                           @ temp1_x = 0xffffffffffffffff
2370
2371         csel    $temp2_x, $temp1_x, $temp0_x, lt
2372         csel    $temp3_x, $temp0_x, xzr, lt
2373
2374         mov     $ctr0.d[1], $temp3_x
2375         mov     $ctr0.d[0], $temp2_x                                    @ ctr0b is mask for last block
2376
2377         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
2378         ext     $h1.16b, $h1.16b, $h1.16b, #8
2379         ld1     { $rk0}, [$output_ptr]                                  @ load existing bytes where the possibly partial last block is to be stored
2380
2381         and     $res1b, $res1b, $ctr0b                                  @ possibly partial last block has zeroes in highest bits
2382
2383         rev64   $res0b, $res1b                                          @ GHASH final block
2384
2385         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
2386
2387         pmull2  $rk2q1, $res0.2d, $h1.2d                                @ GHASH final block - high
2388         ins     $t0.d[0], $res0.d[1]                                    @ GHASH final block - mid
2389
2390         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final block - high
2391         eor     $t0.8b, $t0.8b, $res0.8b                                @ GHASH final block - mid
2392
2393         bif     $res4b, $rk0, $ctr0b                                    @ insert existing bytes in top end of result before storing
2394
2395         pmull   $t0.1q, $t0.1d, $h12k.1d                                @ GHASH final block - mid
2396         st1     { $res4b}, [$output_ptr]                                @ store all 16B
2397
2398         pmull   $rk3q1, $res0.1d, $h1.1d                                @ GHASH final block - low
2399
2400         eor     $acc_mb, $acc_mb, $t0.16b                               @ GHASH final block - mid
2401         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
2402
2403         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final block - low
2404
2405         eor     $t10.16b, $acc_hb, $acc_lb                              @ MODULO - karatsuba tidy up
2406
2407         pmull   $t11.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
2408         ext     $acc_hb, $acc_hb, $acc_hb, #8                           @ MODULO - other top alignment
2409
2410         eor     $acc_mb, $acc_mb, $t10.16b                              @ MODULO - karatsuba tidy up
2411
2412         eor3    $acc_mb, $acc_mb, $acc_hb, $t11.16b                     @ MODULO - fold into mid
2413
2414         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
2415         ext     $acc_mb, $acc_mb, $acc_mb, #8                           @ MODULO - other mid alignment
2416
2417         eor3    $acc_lb, $acc_lb, $acc_mb, $acc_hb                      @ MODULO - fold into low
2418         ext     $acc_lb, $acc_lb, $acc_lb, #8
2419         rev64   $acc_lb, $acc_lb
2420         st1     { $acc_l.16b }, [$current_tag]
2421         rev32   $rtmp_ctr.16b, $rtmp_ctr.16b
2422
2423         str     $rtmp_ctrq, [$counter]                                  @ store the updated counter
2424
2425         lsr     x0, $bit_length, #3
2426
2427         ldp     d10, d11, [sp, #16]
2428         ldp     d12, d13, [sp, #32]
2429         ldp     d14, d15, [sp, #48]
2430         ldp     d8, d9, [sp], #80
2431         ret
2432 .L128_dec_ret:
2433         mov w0, #0x0
2434         ret
2435 .size unroll8_eor3_aes_gcm_dec_128_kernel,.-unroll8_eor3_aes_gcm_dec_128_kernel
2436 ___
2437 }
2438
2439 {
2440 my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7));
2441 my ($temp2_x,$temp3_x)=map("x$_",(13..14));
2442 my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15));
2443 my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15));
2444 my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7));
2445 my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7));
2446 my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15));
2447
2448 my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15));
2449 my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15));
2450 my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15));
2451
2452 my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19));
2453 my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19));
2454
2455 my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25));
2456 my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25));
2457 my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25));
2458 my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25));
2459
2460 my $t0="v16";
2461 my $t0d="d16";
2462
2463 my $t1="v29";
2464 my $t2=$res1;
2465 my $t3=$t1;
2466
2467 my $t4=$res0;
2468 my $t5=$res2;
2469 my $t6=$t0;
2470
2471 my $t7=$res3;
2472 my $t8=$res4;
2473 my $t9=$res5;
2474
2475 my $t10=$res6;
2476 my $t11="v21";
2477 my $t12=$t1;
2478
2479 my $rtmp_ctr="v30";
2480 my $rtmp_ctrq="q30";
2481 my $rctr_inc="v31";
2482 my $rctr_incd="d31";
2483
2484 my $mod_constantd=$t0d;
2485 my $mod_constant=$t0;
2486
2487 my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28));
2488 my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28));
2489 my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28));
2490 my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28));
2491 my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28));
2492 my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28));
2493 my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28));
2494 my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28));
2495 my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28));
2496 my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28));
2497 my $rk2q1="v28.1q";
2498 my $rk3q1="v26.1q";
2499 my $rk4v="v27";
2500
2501 #########################################################################################
2502 # size_t unroll8_eor3_aes_gcm_enc_192_kernel(const unsigned char *in,
2503 #                               size_t len,
2504 #                               unsigned char *out,
2505 #                               const void *key,
2506 #                               unsigned char ivec[16],
2507 #                               u64 *Xi);
2508 #
2509 $code.=<<___;
2510 .global unroll8_eor3_aes_gcm_enc_192_kernel
2511 .type   unroll8_eor3_aes_gcm_enc_192_kernel,%function
2512 .align  4
2513 unroll8_eor3_aes_gcm_enc_192_kernel:
2514         AARCH64_VALID_CALL_TARGET
2515         cbz     x1, .L192_enc_ret
2516         stp     d8, d9, [sp, #-80]!
2517         mov     $counter, x4
2518         mov     $cc, x5
2519         stp     d10, d11, [sp, #16]
2520         stp     d12, d13, [sp, #32]
2521         stp     d14, d15, [sp, #48]
2522         mov     x5, #0xc200000000000000
2523         stp     x5, xzr, [sp, #64]
2524         add     $modulo_constant, sp, #64
2525
2526         lsr     $main_end_input_ptr, $bit_length, #3                    @ byte_len
2527         ld1     { $ctr0b}, [$counter]                                   @ CTR block 0
2528
2529         mov     $constant_temp, #0x100000000                            @ set up counter increment
2530         movi    $rctr_inc.16b, #0x0
2531         mov     $rctr_inc.d[1], $constant_temp
2532
2533         rev32   $rtmp_ctr.16b, $ctr0.16b                                @ set up reversed counter
2534
2535         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 0
2536
2537         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 1
2538         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 1
2539
2540         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 2
2541         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 2
2542
2543         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 3
2544         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 3
2545
2546         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 4
2547         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 4
2548         sub     $main_end_input_ptr, $main_end_input_ptr, #1            @ byte_len - 1
2549
2550         and     $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80   @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
2551
2552         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 5
2553         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 5
2554         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
2555
2556         add     $main_end_input_ptr, $main_end_input_ptr, $input_ptr
2557
2558         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 6
2559         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 6
2560
2561         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 7
2562
2563         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 0
2564         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 0
2565         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 0
2566
2567         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 0
2568         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 0
2569         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 0
2570
2571         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 0
2572         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 0
2573         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
2574
2575         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 1
2576         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 1
2577
2578         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 1
2579         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 1
2580         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 1
2581
2582         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 2
2583         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 1
2584         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 1
2585
2586         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 1
2587         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 2
2588         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 2
2589
2590         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 2
2591         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 2
2592
2593         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 2
2594         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 2
2595         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 2
2596
2597         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
2598         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 3
2599
2600         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 3
2601         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 3
2602         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 3
2603
2604         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 3
2605
2606         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 3
2607
2608         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 3
2609
2610         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 4
2611         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 4
2612         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 3
2613
2614         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 4
2615         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 4
2616         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 4
2617
2618         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 4
2619         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 4
2620         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 4
2621
2622         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 5
2623         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
2624         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 5
2625
2626         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 5
2627         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 5
2628         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 5
2629
2630         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 5
2631         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 5
2632         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 5
2633
2634         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 7
2635
2636         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 6
2637         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 6
2638         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 6
2639
2640         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 6
2641         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 6
2642         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 6
2643
2644         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 6
2645         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 6
2646         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
2647
2648         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 7
2649         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 7
2650
2651         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 7
2652         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 7
2653
2654         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 7
2655         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 7
2656
2657         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 7
2658         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 7
2659
2660         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 8
2661         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 8
2662
2663         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 8
2664         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 8
2665         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 8
2666
2667         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 8
2668         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 8
2669         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 8
2670
2671         add     $end_input_ptr, $input_ptr, $bit_length, lsr #3         @ end_input_ptr
2672         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
2673         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 9
2674
2675         ld1     { $acc_lb}, [$current_tag]
2676         ext     $acc_lb, $acc_lb, $acc_lb, #8
2677         rev64   $acc_lb, $acc_lb
2678         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
2679
2680         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 9
2681         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 9
2682
2683         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 9
2684         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 9
2685
2686         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 9
2687         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 9
2688
2689         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 14 - round 10
2690         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 9
2691         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 11 - round 10
2692
2693         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 9 - round 10
2694         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 13 - round 10
2695         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 12 - round 10
2696
2697         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8 - round 10
2698         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 10 - round 10
2699         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 15 - round 10
2700
2701         aese    $ctr6b, $rk11                                           @ AES block 14 - round 11
2702         aese    $ctr3b, $rk11                                           @ AES block 11 - round 11
2703
2704         aese    $ctr4b, $rk11                                           @ AES block 12 - round 11
2705         aese    $ctr7b, $rk11                                           @ AES block 15 - round 11
2706         ldr     $rk12q, [$cc, #192]                                     @ load rk12
2707
2708         aese    $ctr1b, $rk11                                           @ AES block 9 - round 11
2709         aese    $ctr5b, $rk11                                           @ AES block 13 - round 11
2710
2711         aese    $ctr2b, $rk11                                           @ AES block 10 - round 11
2712         aese    $ctr0b, $rk11                                           @ AES block 8 - round 11
2713         b.ge    .L192_enc_tail                                          @ handle tail
2714
2715         ldp     $ctr_t0q, $ctr_t1q, [$input_ptr], #32                   @ AES block 0, 1 - load plaintext
2716
2717         ldp     $ctr_t2q, $ctr_t3q, [$input_ptr], #32                   @ AES block 2, 3 - load plaintext
2718
2719         ldp     $ctr_t4q, $ctr_t5q, [$input_ptr], #32                   @ AES block 4, 5 - load plaintext
2720
2721         ldp     $ctr_t6q, $ctr_t7q, [$input_ptr], #32                   @ AES block 6, 7 - load plaintext
2722
2723         eor3    $res0b, $ctr_t0b, $ctr0b, $rk12                         @ AES block 0 - result
2724         rev32   $ctr0.16b, $rtmp_ctr.16b                                @ CTR block 8
2725         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8
2726
2727         eor3    $res3b, $ctr_t3b, $ctr3b, $rk12                         @ AES block 3 - result
2728         eor3    $res1b, $ctr_t1b, $ctr1b, $rk12                         @ AES block 1 - result
2729
2730         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 9
2731         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 9
2732         eor3    $res4b, $ctr_t4b, $ctr4b, $rk12                         @ AES block 4 - result
2733
2734         eor3    $res5b, $ctr_t5b, $ctr5b, $rk12                         @ AES block 5 - result
2735         eor3    $res7b, $ctr_t7b, $ctr7b, $rk12                         @ AES block 7 - result
2736         stp     $res0q, $res1q, [$output_ptr], #32                      @ AES block 0, 1 - store result
2737
2738         eor3    $res2b, $ctr_t2b, $ctr2b, $rk12                         @ AES block 2 - result
2739         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 10
2740         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 10
2741
2742         stp     $res2q, $res3q, [$output_ptr], #32                      @ AES block 2, 3 - store result
2743         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
2744
2745         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 11
2746         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 11
2747         eor3    $res6b, $ctr_t6b, $ctr6b, $rk12                         @ AES block 6 - result
2748
2749         stp     $res4q, $res5q, [$output_ptr], #32                      @ AES block 4, 5 - store result
2750
2751         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 12
2752         stp     $res6q, $res7q, [$output_ptr], #32                      @ AES block 6, 7 - store result
2753         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 12
2754
2755         b.ge    .L192_enc_prepretail                                    @ do prepretail
2756
2757 .L192_enc_main_loop:                                                    @ main loop start
2758         rev64   $res4b, $res4b                                          @ GHASH block 8k+4 (t0, t1, and t2 free)
2759         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
2760         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
2761
2762         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
2763         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
2764         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
2765         ext     $h7.16b, $h7.16b, $h7.16b, #8
2766         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
2767         ext     $h8.16b, $h8.16b, $h8.16b, #8
2768
2769         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
2770         rev64   $res0b, $res0b                                          @ GHASH block 8k
2771         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
2772         ext     $h5.16b, $h5.16b, $h5.16b, #8
2773         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
2774         ext     $h6.16b, $h6.16b, $h6.16b, #8
2775
2776         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
2777         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
2778         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
2779
2780         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
2781         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
2782         rev64   $res5b, $res5b                                          @ GHASH block 8k+5 (t0, t1, t2 and t3 free)
2783
2784         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
2785         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
2786         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
2787
2788         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
2789         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
2790         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
2791
2792         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
2793         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
2794         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
2795
2796         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
2797         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
2798         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
2799
2800         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
2801         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
2802         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
2803
2804         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
2805         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
2806         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
2807         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
2808
2809         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
2810         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
2811         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
2812
2813         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
2814         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
2815         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
2816
2817         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
2818         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
2819         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
2820
2821         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
2822         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
2823         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
2824
2825         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
2826         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
2827         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
2828
2829         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
2830         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
2831         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
2832
2833         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
2834         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
2835         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
2836
2837         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
2838         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
2839         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
2840
2841         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
2842         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
2843         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
2844
2845         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
2846         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
2847         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
2848         ext     $h3.16b, $h3.16b, $h3.16b, #8
2849         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
2850         ext     $h4.16b, $h4.16b, $h4.16b, #8
2851
2852         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k - mid
2853         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
2854         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
2855
2856         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
2857         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
2858         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
2859
2860         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
2861         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
2862         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
2863
2864         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
2865         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
2866         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
2867
2868         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
2869         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
2870         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
2871
2872         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
2873         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
2874         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
2875
2876         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
2877         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
2878         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
2879
2880         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
2881         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
2882         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
2883         ext     $h1.16b, $h1.16b, $h1.16b, #8
2884         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
2885         ext     $h2.16b, $h2.16b, $h2.16b, #8
2886
2887         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
2888         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
2889         rev64   $res7b, $res7b                                          @ GHASH block 8k+7 (t0, t1, t2 and t3 free)
2890
2891         rev64   $res6b, $res6b                                          @ GHASH block 8k+6 (t0, t1, and t2 free)
2892         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
2893         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
2894
2895         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
2896         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
2897
2898         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
2899         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
2900         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
2901
2902         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
2903         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
2904         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
2905
2906         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
2907         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
2908         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
2909
2910         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
2911         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
2912         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
2913
2914         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
2915         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
2916         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
2917
2918         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
2919         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
2920         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
2921
2922         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
2923         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
2924
2925         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
2926         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
2927         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
2928
2929         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
2930         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
2931
2932         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
2933         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
2934         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
2935
2936         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
2937         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
2938         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
2939
2940         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
2941         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
2942         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
2943
2944         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
2945         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
2946         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
2947
2948         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
2949         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
2950         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
2951
2952         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
2953         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
2954         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
2955
2956         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
2957         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
2958         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
2959
2960         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
2961         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
2962         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
2963
2964         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
2965         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
2966         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
2967
2968         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
2969         rev32   $h1.16b, $rtmp_ctr.16b                                  @ CTR block 8k+16
2970         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+16
2971
2972         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 9
2973         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
2974         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
2975
2976         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 9
2977         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 9
2978         ldp     $ctr_t0q, $ctr_t1q, [$input_ptr], #32                   @ AES block 8k+8, 8k+9 - load plaintext
2979
2980         pmull   $t11.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
2981         rev32   $h2.16b, $rtmp_ctr.16b                                  @ CTR block 8k+17
2982         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 9
2983
2984         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 9
2985         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 9
2986         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 9
2987
2988         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
2989         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 9
2990         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+17
2991
2992         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 10
2993         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 10
2994         ldr     $rk12q, [$cc, #192]                                     @ load rk12
2995         ext     $t12.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
2996
2997         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 10
2998         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 10
2999         ldp     $ctr_t2q, $ctr_t3q, [$input_ptr], #32                   @ AES block 8k+10, 8k+11 - load plaintext
3000
3001         aese    $ctr4b, $rk11                                           @ AES block 8k+12 - round 11
3002         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
3003         ldp     $ctr_t4q, $ctr_t5q, [$input_ptr], #32                   @ AES block 8k+12, 8k+13 - load plaintext
3004
3005         ldp     $ctr_t6q, $ctr_t7q, [$input_ptr], #32                   @ AES block 8k+14, 8k+15 - load plaintext
3006         aese    $ctr2b, $rk11                                           @ AES block 8k+10 - round 11
3007         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 10
3008
3009         rev32   $h3.16b, $rtmp_ctr.16b                                  @ CTR block 8k+18
3010         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 10
3011
3012         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 10
3013         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
3014
3015         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 10
3016         aese    $ctr5b, $rk11                                           @ AES block 8k+13 - round 11
3017         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+18
3018
3019         aese    $ctr7b, $rk11                                           @ AES block 8k+15 - round 11
3020         aese    $ctr0b, $rk11                                           @ AES block 8k+8 - round 11
3021         eor3    $res4b, $ctr_t4b, $ctr4b, $rk12                         @ AES block 4 - result
3022
3023         aese    $ctr6b, $rk11                                           @ AES block 8k+14 - round 11
3024         aese    $ctr3b, $rk11                                           @ AES block 8k+11 - round 11
3025         aese    $ctr1b, $rk11                                           @ AES block 8k+9 - round 11
3026
3027         rev32   $h4.16b, $rtmp_ctr.16b                                  @ CTR block 8k+19
3028         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+19
3029         eor3    $res7b, $ctr_t7b, $ctr7b, $rk12                         @ AES block 7 - result
3030
3031         eor3    $res2b, $ctr_t2b, $ctr2b, $rk12                         @ AES block 8k+10 - result
3032         eor3    $res0b, $ctr_t0b, $ctr0b, $rk12                         @ AES block 8k+8 - result
3033         mov     $ctr2.16b, $h3.16b                                      @ CTR block 8k+18
3034
3035         eor3    $res1b, $ctr_t1b, $ctr1b, $rk12                         @ AES block 8k+9 - result
3036         mov     $ctr1.16b, $h2.16b                                      @ CTR block 8k+17
3037         stp     $res0q, $res1q, [$output_ptr], #32                      @ AES block 8k+8, 8k+9 - store result
3038         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
3039
3040         eor3    $res6b, $ctr_t6b, $ctr6b, $rk12                         @ AES block 6 - result
3041         mov     $ctr0.16b, $h1.16b                                      @ CTR block 8k+16
3042         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 8k+20
3043
3044         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+20
3045         eor3    $res5b, $ctr_t5b, $ctr5b, $rk12                         @ AES block 5 - result
3046         eor3    $acc_lb, $acc_lb, $acc_hb, $t11.16b                     @ MODULO - fold into low
3047
3048         eor3    $res3b, $ctr_t3b, $ctr3b, $rk12                         @ AES block 8k+11 - result
3049         mov     $ctr3.16b, $h4.16b                                      @ CTR block 8k+19
3050
3051         stp     $res2q, $res3q, [$output_ptr], #32                      @ AES block 8k+10, 8k+11 - store result
3052
3053         stp     $res4q, $res5q, [$output_ptr], #32                      @ AES block 8k+12, 8k+13 - store result
3054
3055         cmp     $input_ptr, $main_end_input_ptr                         @ LOOP CONTROL
3056         stp     $res6q, $res7q, [$output_ptr], #32                      @ AES block 8k+14, 8k+15 - store result
3057         b.lt    .L192_enc_main_loop
3058
3059 .L192_enc_prepretail:                                                   @ PREPRETAIL
3060         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
3061         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
3062         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
3063
3064         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
3065         ext     $h7.16b, $h7.16b, $h7.16b, #8
3066         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
3067         ext     $h8.16b, $h8.16b, $h8.16b, #8
3068         rev64   $res0b, $res0b                                          @ GHASH block 8k
3069         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
3070
3071         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
3072         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
3073         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
3074         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
3075
3076         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
3077         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
3078         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
3079         ext     $h5.16b, $h5.16b, $h5.16b, #8
3080         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
3081         ext     $h6.16b, $h6.16b, $h6.16b, #8
3082
3083         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
3084         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
3085         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
3086
3087         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
3088         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
3089         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
3090
3091         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
3092         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
3093         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
3094
3095         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
3096         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
3097         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
3098
3099         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
3100         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
3101         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
3102
3103         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
3104         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
3105         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
3106
3107         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
3108         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
3109         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
3110
3111         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
3112         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
3113         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
3114
3115         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
3116         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
3117         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
3118
3119         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
3120         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
3121         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
3122
3123         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
3124         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
3125         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
3126
3127         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
3128         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
3129         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
3130
3131         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
3132         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
3133         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
3134
3135         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
3136         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k        - mid
3137         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
3138
3139         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
3140         rev64   $res5b, $res5b                                          @ GHASH block 8k+5 (t0, t1, t2 and t3 free)
3141         rev64   $res6b, $res6b                                          @ GHASH block 8k+6 (t0, t1, and t2 free)
3142
3143         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
3144         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
3145         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
3146
3147         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
3148         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
3149         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
3150
3151         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
3152         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
3153         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
3154
3155         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
3156         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
3157         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
3158
3159         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
3160         ext     $h3.16b, $h3.16b, $h3.16b, #8
3161         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
3162         ext     $h4.16b, $h4.16b, $h4.16b, #8
3163         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
3164         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
3165
3166         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
3167         ext     $h1.16b, $h1.16b, $h1.16b, #8
3168         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
3169         ext     $h2.16b, $h2.16b, $h2.16b, #8
3170         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
3171         rev64   $res4b, $res4b                                          @ GHASH block 8k+4 (t0, t1, and t2 free)
3172
3173         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
3174         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
3175         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
3176
3177         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
3178         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
3179         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
3180
3181         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
3182         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
3183         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
3184
3185         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
3186         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
3187         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
3188
3189         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
3190         rev64   $res7b, $res7b                                          @ GHASH block 8k+7 (t0, t1, t2 and t3 free)
3191         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
3192         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
3193
3194         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
3195         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
3196         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
3197
3198         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
3199         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
3200         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
3201
3202         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
3203         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
3204
3205         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
3206         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
3207         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
3208
3209         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
3210         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
3211         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
3212
3213         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
3214         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
3215         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
3216
3217         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
3218         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
3219         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
3220
3221         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
3222         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
3223
3224         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
3225         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
3226         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
3227
3228         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
3229         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
3230         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
3231
3232         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
3233         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
3234         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
3235
3236         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
3237         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
3238         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
3239
3240         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
3241         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
3242         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
3243
3244         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
3245         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
3246         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
3247
3248         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
3249         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
3250         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
3251
3252         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
3253         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
3254
3255         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
3256         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
3257
3258         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
3259         ext     $t12.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
3260         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
3261         pmull   $t11.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
3262
3263         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
3264         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
3265
3266         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
3267         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
3268         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
3269
3270         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
3271         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 9
3272         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
3273
3274         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
3275         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
3276         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
3277
3278         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 9
3279         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
3280         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 9
3281
3282         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 9
3283         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 9
3284
3285         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
3286         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 9
3287         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 9
3288         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 9
3289
3290         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
3291         ldr     $rk12q, [$cc, #192]                                     @ load rk12
3292
3293         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 10
3294         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 10
3295         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 10
3296
3297         eor3    $acc_lb, $acc_lb, $acc_hb, $t11.16b                     @ MODULO - fold into low
3298         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 10
3299         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 10
3300
3301         aese    $ctr1b, $rk11                                           @ AES block 8k+9 - round 11
3302         aese    $ctr7b, $rk11                                           @ AES block 8k+15 - round 11
3303
3304         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 10
3305         aese    $ctr3b, $rk11                                           @ AES block 8k+11 - round 11
3306
3307         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 10
3308         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 10
3309
3310         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
3311         aese    $ctr2b, $rk11                                           @ AES block 8k+10 - round 11
3312         aese    $ctr0b, $rk11                                           @ AES block 8k+8 - round 11
3313
3314         aese    $ctr6b, $rk11                                           @ AES block 8k+14 - round 11
3315         aese    $ctr4b, $rk11                                           @ AES block 8k+12 - round 11
3316         aese    $ctr5b, $rk11                                           @ AES block 8k+13 - round 11
3317
3318 .L192_enc_tail:                                                         @ TAIL
3319
3320         ldp     $h5q, $h56kq, [$current_tag, #128]                      @ load h5l | h5h
3321         ext     $h5.16b, $h5.16b, $h5.16b, #8
3322         sub     $main_end_input_ptr, $end_input_ptr, $input_ptr         @ main_end_input_ptr is number of bytes left to process
3323
3324         ldr     $ctr_t0q, [$input_ptr], #16                             @ AES block 8k+8 - l3ad plaintext
3325
3326         ldp     $h78kq, $h8q, [$current_tag, #192]                      @ load h8k | h7k
3327         ext     $h8.16b, $h8.16b, $h8.16b, #8
3328
3329         mov     $t1.16b, $rk12
3330
3331         ldp     $h6q, $h7q, [$current_tag, #160]                        @ load h6l | h6h
3332         ext     $h6.16b, $h6.16b, $h6.16b, #8
3333         ext     $h7.16b, $h7.16b, $h7.16b, #8
3334         cmp     $main_end_input_ptr, #112
3335
3336         eor3    $res1b, $ctr_t0b, $ctr0b, $t1.16b                       @ AES block 8k+8 - result
3337         ext     $t0.16b, $acc_lb, $acc_lb, #8                           @ prepare final partial tag
3338         b.gt    .L192_enc_blocks_more_than_7
3339
3340         cmp     $main_end_input_ptr, #96
3341         mov     $ctr7b, $ctr6b
3342         movi    $acc_h.8b, #0
3343
3344         mov     $ctr6b, $ctr5b
3345         movi    $acc_l.8b, #0
3346         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
3347
3348         mov     $ctr5b, $ctr4b
3349         mov     $ctr4b, $ctr3b
3350         mov     $ctr3b, $ctr2b
3351
3352         mov     $ctr2b, $ctr1b
3353         movi    $acc_m.8b, #0
3354         b.gt    .L192_enc_blocks_more_than_6
3355
3356         mov     $ctr7b, $ctr6b
3357         cmp     $main_end_input_ptr, #80
3358
3359         mov     $ctr6b, $ctr5b
3360         mov     $ctr5b, $ctr4b
3361         mov     $ctr4b, $ctr3b
3362
3363         mov     $ctr3b, $ctr1b
3364         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
3365         b.gt    .L192_enc_blocks_more_than_5
3366
3367         cmp     $main_end_input_ptr, #64
3368         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
3369
3370         mov     $ctr7b, $ctr6b
3371         mov     $ctr6b, $ctr5b
3372         mov     $ctr5b, $ctr4b
3373
3374         mov     $ctr4b, $ctr1b
3375         b.gt    .L192_enc_blocks_more_than_4
3376
3377         mov     $ctr7b, $ctr6b
3378         mov     $ctr6b, $ctr5b
3379         mov     $ctr5b, $ctr1b
3380
3381         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
3382         cmp     $main_end_input_ptr, #48
3383         b.gt    .L192_enc_blocks_more_than_3
3384
3385         mov     $ctr7b, $ctr6b
3386         mov     $ctr6b, $ctr1b
3387         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
3388
3389         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
3390         cmp     $main_end_input_ptr, #32
3391         b.gt    .L192_enc_blocks_more_than_2
3392
3393         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
3394
3395         cmp     $main_end_input_ptr, #16
3396         mov     $ctr7b, $ctr1b
3397         b.gt    .L192_enc_blocks_more_than_1
3398
3399         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
3400         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
3401         b        .L192_enc_blocks_less_than_1
3402 .L192_enc_blocks_more_than_7:                                           @ blocks left >  7
3403         st1     { $res1b}, [$output_ptr], #16                           @ AES final-7 block  - store result
3404
3405         rev64   $res0b, $res1b                                          @ GHASH final-7 block
3406         ins     $acc_m.d[0], $h78k.d[1]                                 @ GHASH final-7 block - mid
3407
3408         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
3409
3410         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-7 block - mid
3411
3412         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-6 block - load plaintext
3413
3414         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-7 block - mid
3415         movi    $t0.8b, #0                                              @ supress further partial tag feed in
3416         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH final-7 block - low
3417
3418         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH final-7 block - high
3419
3420         pmull   $acc_m.1q, $rk4v.1d, $acc_m.1d                          @ GHASH final-7 block - mid
3421         eor3    $res1b, $ctr_t1b, $ctr1b, $t1.16b                       @ AES final-6 block - result
3422 .L192_enc_blocks_more_than_6:                                           @ blocks left >  6
3423
3424         st1     { $res1b}, [$output_ptr], #16                           @ AES final-6 block - store result
3425
3426         rev64   $res0b, $res1b                                          @ GHASH final-6 block
3427
3428         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-5 block - load plaintext
3429
3430         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
3431
3432         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-6 block - mid
3433
3434         pmull   $rk3q1, $res0.1d, $h7.1d                                @ GHASH final-6 block - low
3435         eor3    $res1b, $ctr_t1b, $ctr2b, $t1.16b                       @ AES final-5 block - result
3436
3437         movi    $t0.8b, #0                                              @ supress further partial tag feed in
3438         pmull2  $rk2q1, $res0.2d, $h7.2d                                @ GHASH final-6 block - high
3439         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-6 block - mid
3440
3441         pmull   $rk4v.1q, $rk4v.1d, $h78k.1d                            @ GHASH final-6 block - mid
3442
3443         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-6 block - high
3444         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-6 block - low
3445
3446         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-6 block - mid
3447 .L192_enc_blocks_more_than_5:                                           @ blocks left >  5
3448
3449         st1     { $res1b}, [$output_ptr], #16                           @ AES final-5 block - store result
3450
3451         rev64   $res0b, $res1b                                          @ GHASH final-5 block
3452
3453         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
3454
3455         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-5 block - mid
3456
3457         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-4 block - load plaintext
3458         pmull2  $rk2q1, $res0.2d, $h6.2d                                @ GHASH final-5 block - high
3459
3460         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-5 block - mid
3461         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-5 block - high
3462
3463         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-5 block - mid
3464         pmull   $rk3q1, $res0.1d, $h6.1d                                @ GHASH final-5 block - low
3465
3466         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-5 block - low
3467         pmull2  $rk4v.1q, $rk4v.2d, $h56k.2d                            @ GHASH final-5 block - mid
3468
3469         eor3    $res1b, $ctr_t1b, $ctr3b, $t1.16b                       @ AES final-4 block - result
3470         movi    $t0.8b, #0                                              @ supress further partial tag feed in
3471
3472         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-5 block - mid
3473 .L192_enc_blocks_more_than_4:                                           @ blocks left >  4
3474
3475         st1     { $res1b}, [$output_ptr], #16                           @ AES final-4 block - store result
3476
3477         rev64   $res0b, $res1b                                          @ GHASH final-4 block
3478
3479         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
3480
3481         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-3 block - load plaintext
3482         pmull2  $rk2q1, $res0.2d, $h5.2d                                @ GHASH final-4 block - high
3483         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-4 block - mid
3484
3485         pmull   $rk3q1, $res0.1d, $h5.1d                                @ GHASH final-4 block - low
3486         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-4 block - high
3487
3488         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-4 block - mid
3489
3490         movi    $t0.8b, #0                                              @ supress further partial tag feed in
3491         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-4 block - low
3492
3493         pmull   $rk4v.1q, $rk4v.1d, $h56k.1d                            @ GHASH final-4 block - mid
3494
3495         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-4 block - mid
3496         eor3    $res1b, $ctr_t1b, $ctr4b, $t1.16b                       @ AES final-3 block - result
3497 .L192_enc_blocks_more_than_3:                                           @ blocks left >  3
3498
3499         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
3500         st1     { $res1b}, [$output_ptr], #16                           @ AES final-3 block - store result
3501
3502         rev64   $res0b, $res1b                                          @ GHASH final-3 block
3503
3504         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
3505         movi    $t0.8b, #0                                              @ supress further partial tag feed in
3506
3507         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-2 block - load plaintext
3508         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
3509         ext     $h4.16b, $h4.16b, $h4.16b, #8
3510
3511         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-3 block - mid
3512
3513         eor3    $res1b, $ctr_t1b, $ctr5b, $t1.16b                       @ AES final-2 block - result
3514         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-3 block - mid
3515
3516         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-3 block - mid
3517         pmull   $rk3q1, $res0.1d, $h4.1d                                @ GHASH final-3 block - low
3518
3519         pmull2  $rk2q1, $res0.2d, $h4.2d                                @ GHASH final-3 block - high
3520         pmull2  $rk4v.1q, $rk4v.2d, $h34k.2d                            @ GHASH final-3 block - mid
3521
3522         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-3 block - low
3523
3524         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-3 block - mid
3525         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-3 block - high
3526 .L192_enc_blocks_more_than_2:                                           @ blocks left >  2
3527
3528         st1     { $res1b}, [$output_ptr], #16                           @ AES final-2 block - store result
3529
3530         rev64   $res0b, $res1b                                          @ GHASH final-2 block
3531         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
3532         ext     $h3.16b, $h3.16b, $h3.16b, #8
3533
3534         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
3535
3536         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-1 block - load plaintext
3537         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-2 block - mid
3538
3539         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-2 block - mid
3540
3541         pmull   $rk3q1, $res0.1d, $h3.1d                                @ GHASH final-2 block - low
3542         pmull2  $rk2q1, $res0.2d, $h3.2d                                @ GHASH final-2 block - high
3543         movi    $t0.8b, #0                                              @ supress further partial tag feed in
3544
3545         pmull   $rk4v.1q, $rk4v.1d, $h34k.1d                            @ GHASH final-2 block - mid
3546
3547         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-2 block - low
3548         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-2 block - high
3549
3550         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-2 block - mid
3551         eor3    $res1b, $ctr_t1b, $ctr6b, $t1.16b                       @ AES final-1 block - result
3552 .L192_enc_blocks_more_than_1:                                           @ blocks left >  1
3553
3554         ldr     $h2q, [$current_tag, #64]                               @ load h1l | h1h
3555         ext     $h2.16b, $h2.16b, $h2.16b, #8
3556         st1     { $res1b}, [$output_ptr], #16                           @ AES final-1 block - store result
3557
3558         rev64   $res0b, $res1b                                          @ GHASH final-1 block
3559
3560         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
3561
3562         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-1 block - mid
3563         pmull   $rk3q1, $res0.1d, $h2.1d                                @ GHASH final-1 block - low
3564
3565         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-1 block - low
3566         pmull2  $rk2q1, $res0.2d, $h2.2d                                @ GHASH final-1 block - high
3567         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-1 block - mid
3568
3569         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final block - load plaintext
3570         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
3571
3572         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-1 block - mid
3573
3574         eor3    $res1b, $ctr_t1b, $ctr7b, $t1.16b                       @ AES final block - result
3575         pmull2  $rk4v.1q, $rk4v.2d, $h12k.2d                            @ GHASH final-1 block - mid
3576
3577         movi    $t0.8b, #0                                              @ supress further partial tag feed in
3578
3579         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-1 block - mid
3580         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-1 block - high
3581 .L192_enc_blocks_less_than_1:                                           @ blocks left <= 1
3582
3583         mvn     $temp0_x, xzr                                           @ temp0_x = 0xffffffffffffffff
3584         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
3585
3586         sub     $bit_length, $bit_length, #128                          @ bit_length -= 128
3587
3588         neg     $bit_length, $bit_length                                @ bit_length = 128 - #bits in input (in range [1,128])
3589
3590         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
3591
3592         lsr     $temp0_x, $temp0_x, $bit_length                         @ temp0_x is mask for top 64b of last block
3593         cmp     $bit_length, #64
3594         mvn     $temp1_x, xzr                                           @ temp1_x = 0xffffffffffffffff
3595
3596         csel    $temp2_x, $temp1_x, $temp0_x, lt
3597         csel    $temp3_x, $temp0_x, xzr, lt
3598
3599         mov     $ctr0.d[1], $temp3_x
3600         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
3601         ext     $h1.16b, $h1.16b, $h1.16b, #8
3602
3603         ld1     { $rk0}, [$output_ptr]                                  @ load existing bytes where the possibly partial last block is to be stored
3604         mov     $ctr0.d[0], $temp2_x                                    @ ctr0b is mask for last block
3605
3606         and     $res1b, $res1b, $ctr0b                                  @ possibly partial last block has zeroes in highest bits
3607
3608         rev64   $res0b, $res1b                                          @ GHASH final block
3609         bif     $res1b, $rk0, $ctr0b                                    @ insert existing bytes in top end of result before storing
3610
3611         st1     { $res1b}, [$output_ptr]                                @ store all 16B
3612
3613         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
3614
3615         ins     $t0.d[0], $res0.d[1]                                    @ GHASH final block - mid
3616         pmull2  $rk2q1, $res0.2d, $h1.2d                                @ GHASH final block - high
3617
3618         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final block - high
3619         pmull   $rk3q1, $res0.1d, $h1.1d                                @ GHASH final block - low
3620
3621         eor     $t0.8b, $t0.8b, $res0.8b                                @ GHASH final block - mid
3622
3623         pmull   $t0.1q, $t0.1d, $h12k.1d                                @ GHASH final block - mid
3624
3625         eor     $acc_mb, $acc_mb, $t0.16b                               @ GHASH final block - mid
3626         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
3627
3628         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final block - low
3629         ext     $t11.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
3630
3631         rev32   $rtmp_ctr.16b, $rtmp_ctr.16b
3632
3633         str     $rtmp_ctrq, [$counter]                                  @ store the updated counter
3634         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
3635
3636         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
3637
3638         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
3639
3640         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
3641         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
3642
3643         eor3    $acc_lb, $acc_lb, $acc_hb, $t11.16b                     @ MODULO - fold into low
3644                 ext     $acc_lb, $acc_lb, $acc_lb, #8
3645         rev64   $acc_lb, $acc_lb
3646         st1     { $acc_l.16b }, [$current_tag]
3647
3648         lsr     x0, $bit_length, #3                                     @ return sizes
3649
3650         ldp     d10, d11, [sp, #16]
3651         ldp     d12, d13, [sp, #32]
3652         ldp     d14, d15, [sp, #48]
3653         ldp     d8, d9, [sp], #80
3654         ret
3655
3656 .L192_enc_ret:
3657         mov w0, #0x0
3658         ret
3659 .size unroll8_eor3_aes_gcm_enc_192_kernel,.-unroll8_eor3_aes_gcm_enc_192_kernel
3660 ___
3661
3662 #########################################################################################
3663 # size_t unroll8_eor3_aes_gcm_dec_192_kernel(const unsigned char *in,
3664 #                               size_t len,
3665 #                               unsigned char *out,
3666 #                               const void *key,
3667 #                               unsigned char ivec[16],
3668 #                               u64 *Xi);
3669 #
3670 $code.=<<___;
3671 .global unroll8_eor3_aes_gcm_dec_192_kernel
3672 .type   unroll8_eor3_aes_gcm_dec_192_kernel,%function
3673 .align  4
3674 unroll8_eor3_aes_gcm_dec_192_kernel:
3675         AARCH64_VALID_CALL_TARGET
3676         cbz     x1, .L192_dec_ret
3677         stp     d8, d9, [sp, #-80]!
3678         mov     $counter, x4
3679         mov     $cc, x5
3680         stp     d10, d11, [sp, #16]
3681         stp     d12, d13, [sp, #32]
3682         stp     d14, d15, [sp, #48]
3683         mov     x5, #0xc200000000000000
3684         stp     x5, xzr, [sp, #64]
3685         add     $modulo_constant, sp, #64
3686
3687         lsr     $main_end_input_ptr, $bit_length, #3                    @ byte_len
3688         ld1     { $ctr0b}, [$counter]                                   @ CTR block 0
3689         ld1     { $acc_lb}, [$current_tag]
3690
3691                 mov     $constant_temp, #0x100000000                    @ set up counter increment
3692         movi    $rctr_inc.16b, #0x0
3693         mov     $rctr_inc.d[1], $constant_temp
3694
3695         rev32   $rtmp_ctr.16b, $ctr0.16b                                @ set up reversed counter
3696
3697         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 0
3698
3699         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 1
3700         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 1
3701
3702         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 2
3703         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 2
3704
3705         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 3
3706         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 3
3707
3708         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 4
3709         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 4
3710
3711         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 5
3712         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 5
3713         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
3714
3715         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 6
3716         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 6
3717
3718         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 7
3719
3720         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 0
3721         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 0
3722         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 0
3723
3724         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 0
3725         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 0
3726         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 0
3727
3728         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 0
3729         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 0
3730         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
3731
3732         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 1
3733
3734         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 1
3735
3736         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 1
3737         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 1
3738         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 1
3739
3740         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 1
3741         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 1
3742
3743         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 2
3744         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 2
3745         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 1
3746
3747         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 2
3748         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 2
3749         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 2
3750
3751         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 2
3752         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 2
3753         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 2
3754
3755         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 3
3756
3757         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
3758         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 3
3759         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 3
3760
3761         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 3
3762         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 3
3763
3764         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 3
3765         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 3
3766         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 3
3767
3768         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 4
3769         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 4
3770         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 4
3771
3772         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 4
3773         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 4
3774         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 4
3775
3776         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 4
3777         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 5
3778         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 4
3779
3780         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 5
3781         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
3782
3783         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 5
3784         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 5
3785         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 5
3786
3787         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 5
3788         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 5
3789         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 5
3790
3791         sub     $main_end_input_ptr, $main_end_input_ptr, #1            @ byte_len - 1
3792
3793         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 6
3794         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 6
3795         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 6
3796
3797         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 6
3798         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 6
3799         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 6
3800
3801         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 6
3802         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 6
3803         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
3804
3805         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 7
3806
3807         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 7
3808         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 7
3809
3810         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 7
3811         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 7
3812         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 7
3813
3814         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 7
3815         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 7
3816         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 7
3817
3818         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 8
3819         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 8
3820         and     $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80   @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
3821
3822         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 8
3823         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 8
3824         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 8
3825
3826         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 8
3827         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 8
3828         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 8
3829
3830         add     $end_input_ptr, $input_ptr, $bit_length, lsr #3         @ end_input_ptr
3831         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 9
3832
3833         ld1     { $acc_lb}, [$current_tag]
3834         ext     $acc_lb, $acc_lb, $acc_lb, #8
3835         rev64   $acc_lb, $acc_lb
3836
3837         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
3838
3839         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 9
3840         add     $main_end_input_ptr, $main_end_input_ptr, $input_ptr
3841
3842         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 9
3843         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 9
3844         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 9
3845
3846         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
3847         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 9
3848
3849         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 9
3850         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 9
3851
3852         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 10
3853         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 10
3854         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 10
3855
3856         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 10
3857         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 10
3858         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 10
3859
3860         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 10
3861         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 10
3862         ldr     $rk12q, [$cc, #192]                                     @ load rk12
3863
3864         aese    $ctr0b, $rk11                                           @ AES block 0 - round 11
3865         aese    $ctr1b, $rk11                                           @ AES block 1 - round 11
3866         aese    $ctr4b, $rk11                                           @ AES block 4 - round 11
3867
3868         aese    $ctr6b, $rk11                                           @ AES block 6 - round 11
3869         aese    $ctr5b, $rk11                                           @ AES block 5 - round 11
3870         aese    $ctr7b, $rk11                                           @ AES block 7 - round 11
3871
3872         aese    $ctr2b, $rk11                                           @ AES block 2 - round 11
3873         aese    $ctr3b, $rk11                                           @ AES block 3 - round 11
3874         b.ge    .L192_dec_tail                                          @ handle tail
3875
3876         ldp     $res0q, $res1q, [$input_ptr], #32                       @ AES block 0, 1 - load ciphertext
3877
3878         ldp     $res2q, $res3q, [$input_ptr], #32                       @ AES block 2, 3 - load ciphertext
3879
3880         ldp     $res4q, $res5q, [$input_ptr], #32                       @ AES block 4, 5 - load ciphertext
3881
3882         eor3    $ctr1b, $res1b, $ctr1b, $rk12                           @ AES block 1 - result
3883         eor3    $ctr0b, $res0b, $ctr0b, $rk12                           @ AES block 0 - result
3884         stp     $ctr0q, $ctr1q, [$output_ptr], #32                      @ AES block 0, 1 - store result
3885
3886         rev32   $ctr0.16b, $rtmp_ctr.16b                                @ CTR block 8
3887         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8
3888
3889         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 9
3890         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 9
3891         eor3    $ctr3b, $res3b, $ctr3b, $rk12                           @ AES block 3 - result
3892
3893         eor3    $ctr2b, $res2b, $ctr2b, $rk12                           @ AES block 2 - result
3894         stp     $ctr2q, $ctr3q, [$output_ptr], #32                      @ AES block 2, 3 - store result
3895         ldp     $res6q, $res7q, [$input_ptr], #32                       @ AES block 6, 7 - load ciphertext
3896
3897         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 10
3898         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 10
3899
3900         eor3    $ctr4b, $res4b, $ctr4b, $rk12                           @ AES block 4 - result
3901
3902         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 11
3903         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 11
3904
3905         eor3    $ctr5b, $res5b, $ctr5b, $rk12                           @ AES block 5 - result
3906         stp     $ctr4q, $ctr5q, [$output_ptr], #32                      @ AES block 4, 5 - store result
3907         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
3908
3909         eor3    $ctr6b, $res6b, $ctr6b, $rk12                           @ AES block 6 - result
3910         eor3    $ctr7b, $res7b, $ctr7b, $rk12                           @ AES block 7 - result
3911         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 12
3912
3913         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 12
3914         stp     $ctr6q, $ctr7q, [$output_ptr], #32                      @ AES block 6, 7 - store result
3915         b.ge    .L192_dec_prepretail                                    @ do prepretail
3916
3917 .L192_dec_main_loop:                                                    @ main loop start
3918         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
3919         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
3920         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
3921
3922         rev64   $res0b, $res0b                                          @ GHASH block 8k
3923         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
3924         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
3925
3926         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
3927         ext     $h7.16b, $h7.16b, $h7.16b, #8
3928         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
3929         ext     $h8.16b, $h8.16b, $h8.16b, #8
3930         rev64   $res4b, $res4b                                          @ GHASH block 8k+4
3931         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
3932
3933         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
3934         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
3935         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
3936
3937         rev64   $res5b, $res5b                                          @ GHASH block 8k+5
3938
3939         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
3940         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
3941         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
3942
3943         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
3944         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
3945         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
3946
3947         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
3948         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
3949         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
3950
3951         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
3952         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
3953         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
3954
3955         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
3956         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
3957         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
3958         ext     $h5.16b, $h5.16b, $h5.16b, #8
3959         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
3960         ext     $h6.16b, $h6.16b, $h6.16b, #8
3961
3962         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
3963         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
3964         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
3965
3966         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
3967         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
3968         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
3969
3970         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
3971         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
3972         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
3973
3974         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
3975         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
3976         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
3977         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
3978
3979         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
3980         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
3981         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
3982
3983         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
3984         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
3985         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
3986
3987         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
3988         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
3989         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
3990
3991         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
3992         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
3993         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
3994
3995         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
3996         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
3997         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
3998
3999         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
4000         ext     $h3.16b, $h3.16b, $h3.16b, #8
4001         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
4002         ext     $h4.16b, $h4.16b, $h4.16b, #8
4003         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
4004         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
4005
4006         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
4007         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
4008         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
4009
4010         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
4011         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
4012
4013         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
4014         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
4015         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
4016
4017         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
4018         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
4019         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
4020
4021         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
4022         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
4023
4024         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
4025         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k        - mid
4026         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
4027
4028         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
4029         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
4030         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
4031
4032         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
4033         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
4034         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
4035
4036         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
4037         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
4038         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
4039
4040         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
4041         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
4042         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
4043
4044         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
4045         ext     $h1.16b, $h1.16b, $h1.16b, #8
4046         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
4047         ext     $h2.16b, $h2.16b, $h2.16b, #8
4048         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
4049         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
4050
4051         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
4052         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
4053         rev64   $res7b, $res7b                                          @ GHASH block 8k+7
4054
4055         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
4056         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
4057         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
4058
4059         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
4060         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
4061         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
4062
4063         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
4064         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
4065         rev64   $res6b, $res6b                                          @ GHASH block 8k+6
4066
4067         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
4068         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
4069         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
4070         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
4071
4072         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
4073         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
4074         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
4075
4076         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
4077         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
4078         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
4079
4080         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
4081         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
4082         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
4083
4084         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
4085         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
4086         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
4087
4088         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
4089         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
4090         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
4091
4092         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
4093         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
4094         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
4095
4096         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
4097         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
4098         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
4099
4100         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
4101         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
4102         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
4103
4104         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
4105         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
4106         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
4107
4108         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
4109         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
4110         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
4111
4112         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
4113         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
4114         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
4115
4116         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
4117         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
4118         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
4119
4120         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
4121         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
4122         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
4123
4124         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
4125         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
4126         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
4127
4128         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
4129         rev32   $h1.16b, $rtmp_ctr.16b                                  @ CTR block 8k+16
4130         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+16
4131
4132         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 9
4133         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
4134         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 9
4135
4136         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 9
4137         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 9
4138         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
4139
4140         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
4141         ldp     $res0q, $res1q, [$input_ptr], #32                       @ AES block 8k+8, 8k+9 - load ciphertext
4142
4143         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 9
4144         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 9
4145         ldp     $res2q, $res3q, [$input_ptr], #32                       @ AES block 8k+10, 8k+11 - load ciphertext
4146
4147         rev32   $h2.16b, $rtmp_ctr.16b                                  @ CTR block 8k+17
4148         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
4149         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+17
4150
4151         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 9
4152         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 9
4153         ext     $t11.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
4154
4155         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 10
4156         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 10
4157         ldp     $res4q, $res5q, [$input_ptr], #32                       @ AES block 8k+12, 8k+13 - load ciphertext
4158
4159         rev32   $h3.16b, $rtmp_ctr.16b                                  @ CTR block 8k+18
4160         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+18
4161         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
4162
4163         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 10
4164         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 10
4165         ldr     $rk12q, [$cc, #192]                                     @ load rk12
4166
4167         ldp     $res6q, $res7q, [$input_ptr], #32                       @ AES block 8k+14, 8k+15 - load ciphertext
4168         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 10
4169         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 10
4170
4171         aese    $ctr0b, $rk11                                           @ AES block 8k+8 - round 11
4172         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
4173         aese    $ctr1b, $rk11                                           @ AES block 8k+9 - round 11
4174
4175         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 10
4176         aese    $ctr6b, $rk11                                           @ AES block 8k+14 - round 11
4177         aese    $ctr3b, $rk11                                           @ AES block 8k+11 - round 11
4178
4179         eor3    $ctr0b, $res0b, $ctr0b, $rk12                           @ AES block 8k+8 - result
4180         rev32   $h4.16b, $rtmp_ctr.16b                                  @ CTR block 8k+19
4181         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 10
4182
4183         aese    $ctr4b, $rk11                                           @ AES block 8k+12 - round 11
4184         aese    $ctr2b, $rk11                                           @ AES block 8k+10 - round 11
4185         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+19
4186
4187         aese    $ctr7b, $rk11                                           @ AES block 8k+15 - round 11
4188         aese    $ctr5b, $rk11                                           @ AES block 8k+13 - round 11
4189         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
4190
4191         eor3    $ctr1b, $res1b, $ctr1b, $rk12                           @ AES block 8k+9 - result
4192         stp     $ctr0q, $ctr1q, [$output_ptr], #32                      @ AES block 8k+8, 8k+9 - store result
4193         eor3    $ctr3b, $res3b, $ctr3b, $rk12                           @ AES block 8k+11 - result
4194
4195         eor3    $ctr2b, $res2b, $ctr2b, $rk12                           @ AES block 8k+10 - result
4196         eor3    $ctr7b, $res7b, $ctr7b, $rk12                           @ AES block 8k+15 - result
4197         stp     $ctr2q, $ctr3q, [$output_ptr], #32                      @ AES block 8k+10, 8k+11 - store result
4198
4199         eor3    $ctr5b, $res5b, $ctr5b, $rk12                           @ AES block 8k+13 - result
4200         eor3    $acc_lb, $acc_lb, $acc_hb, $t11.16b                     @ MODULO - fold into low
4201         mov     $ctr3.16b, $h4.16b                                      @ CTR block 8k+19
4202
4203         eor3    $ctr4b, $res4b, $ctr4b, $rk12                           @ AES block 8k+12 - result
4204         stp     $ctr4q, $ctr5q, [$output_ptr], #32                      @ AES block 8k+12, 8k+13 - store result
4205         cmp     $input_ptr, $main_end_input_ptr                         @ LOOP CONTROL
4206
4207         eor3    $ctr6b, $res6b, $ctr6b, $rk12                           @ AES block 8k+14 - result
4208         stp     $ctr6q, $ctr7q, [$output_ptr], #32                      @ AES block 8k+14, 8k+15 - store result
4209         mov     $ctr0.16b, $h1.16b                                      @ CTR block 8k+16
4210
4211         mov     $ctr1.16b, $h2.16b                                      @ CTR block 8k+17
4212         mov     $ctr2.16b, $h3.16b                                      @ CTR block 8k+18
4213
4214         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 8k+20
4215         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+20
4216         b.lt    .L192_dec_main_loop
4217
4218 .L192_dec_prepretail:                                                   @ PREPRETAIL
4219         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
4220         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
4221         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
4222
4223         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
4224         ext     $h7.16b, $h7.16b, $h7.16b, #8
4225         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
4226         ext     $h8.16b, $h8.16b, $h8.16b, #8
4227         rev64   $res0b, $res0b                                          @ GHASH block 8k
4228         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
4229
4230         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
4231         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
4232         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
4233
4234         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
4235         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
4236         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
4237
4238         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
4239         ext     $h5.16b, $h5.16b, $h5.16b, #8
4240         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
4241         ext     $h6.16b, $h6.16b, $h6.16b, #8
4242         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
4243
4244         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
4245         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
4246         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
4247
4248         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
4249         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
4250         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
4251
4252         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
4253         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
4254         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
4255
4256         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
4257         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
4258         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
4259
4260         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
4261         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
4262         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
4263
4264         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
4265         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
4266         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
4267
4268         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
4269         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
4270         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
4271
4272         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
4273         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
4274         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
4275
4276         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
4277         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
4278         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
4279
4280         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
4281         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
4282         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
4283         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
4284
4285         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
4286         rev64   $res5b, $res5b                                          @ GHASH block 8k+5
4287         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
4288
4289         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
4290         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
4291         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
4292
4293         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
4294         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
4295         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
4296
4297         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
4298         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
4299         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
4300
4301         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k        - mid
4302         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
4303         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
4304
4305         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
4306         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
4307         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
4308
4309         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
4310         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
4311         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
4312
4313         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
4314         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
4315         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
4316
4317         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
4318         ext     $h3.16b, $h3.16b, $h3.16b, #8
4319         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
4320         ext     $h4.16b, $h4.16b, $h4.16b, #8
4321         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
4322         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
4323
4324         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
4325         ext     $h1.16b, $h1.16b, $h1.16b, #8
4326         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
4327         ext     $h2.16b, $h2.16b, $h2.16b, #8
4328         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
4329         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
4330
4331         rev64   $res7b, $res7b                                          @ GHASH block 8k+7
4332
4333         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
4334         rev64   $res4b, $res4b                                          @ GHASH block 8k+4
4335
4336         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
4337         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
4338         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
4339
4340         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
4341         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
4342         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
4343
4344         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
4345         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
4346         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
4347
4348         rev64   $res6b, $res6b                                          @ GHASH block 8k+6
4349         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
4350         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
4351         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
4352
4353         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
4354         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
4355         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
4356
4357         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
4358         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
4359         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
4360
4361         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
4362         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
4363         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
4364
4365         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
4366
4367         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
4368         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
4369         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
4370
4371         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
4372         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
4373         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
4374
4375         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
4376         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
4377         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
4378
4379         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
4380         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
4381
4382         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
4383         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
4384         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
4385
4386         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
4387         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
4388         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
4389
4390         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
4391         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
4392         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
4393
4394         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
4395         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
4396         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
4397
4398         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
4399         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
4400         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
4401
4402         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
4403         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
4404         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
4405
4406         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
4407         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
4408         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
4409
4410         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
4411         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
4412         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
4413
4414         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
4415         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
4416         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
4417
4418         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
4419         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
4420
4421         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
4422         ext     $t11.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
4423         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
4424
4425         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
4426         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
4427         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
4428
4429         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
4430         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
4431         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
4432
4433         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
4434         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
4435         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
4436
4437         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
4438         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 9
4439         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 9
4440
4441         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 9
4442         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 9
4443         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 9
4444
4445         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 9
4446         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 9
4447         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 9
4448
4449         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
4450         ldr     $rk12q, [$cc, #192]                                     @ load rk12
4451         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
4452
4453         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 10
4454         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 10
4455         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 10
4456
4457         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 10
4458         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 10
4459         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 10
4460
4461         aese    $ctr0b, $rk11                                           @ AES block 8k+8 - round 11
4462         eor3    $acc_lb, $acc_lb, $acc_hb, $t11.16b                     @ MODULO - fold into low
4463         aese    $ctr5b, $rk11                                           @ AES block 8k+13 - round 11
4464
4465         aese    $ctr2b, $rk11                                           @ AES block 8k+10 - round 11
4466         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 10
4467         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 10
4468
4469         aese    $ctr6b, $rk11                                           @ AES block 8k+14 - round 11
4470         aese    $ctr4b, $rk11                                           @ AES block 8k+12 - round 11
4471         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
4472
4473         aese    $ctr3b, $rk11                                           @ AES block 8k+11 - round 11
4474         aese    $ctr1b, $rk11                                           @ AES block 8k+9 - round 11
4475         aese    $ctr7b, $rk11                                           @ AES block 8k+15 - round 11
4476
4477 .L192_dec_tail:                                                         @ TAIL
4478
4479         sub     $main_end_input_ptr, $end_input_ptr, $input_ptr         @ main_end_input_ptr is number of bytes left to process
4480
4481         ldp     $h5q, $h56kq, [$current_tag, #128]                      @ load h5l | h5h
4482         ext     $h5.16b, $h5.16b, $h5.16b, #8
4483         ldr     $res1q, [$input_ptr], #16                               @ AES block 8k+8 - load ciphertext
4484
4485         ldp     $h78kq, $h8q, [$current_tag, #192]                      @ load h8k | h7k
4486         ext     $h8.16b, $h8.16b, $h8.16b, #8
4487
4488         mov     $t1.16b, $rk12
4489
4490         ldp     $h6q, $h7q, [$current_tag, #160]                        @ load h6l | h6h
4491         ext     $h6.16b, $h6.16b, $h6.16b, #8
4492         ext     $h7.16b, $h7.16b, $h7.16b, #8
4493         ext     $t0.16b, $acc_lb, $acc_lb, #8                           @ prepare final partial tag
4494
4495         eor3    $res4b, $res1b, $ctr0b, $t1.16b                         @ AES block 8k+8 - result
4496         cmp     $main_end_input_ptr, #112
4497         b.gt    .L192_dec_blocks_more_than_7
4498
4499         mov     $ctr7b, $ctr6b
4500         movi    $acc_h.8b, #0
4501         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
4502
4503         mov     $ctr6b, $ctr5b
4504         mov     $ctr5b, $ctr4b
4505         mov     $ctr4b, $ctr3b
4506
4507         cmp     $main_end_input_ptr, #96
4508         movi    $acc_l.8b, #0
4509         mov     $ctr3b, $ctr2b
4510
4511         mov     $ctr2b, $ctr1b
4512         movi    $acc_m.8b, #0
4513         b.gt    .L192_dec_blocks_more_than_6
4514
4515         mov     $ctr7b, $ctr6b
4516         mov     $ctr6b, $ctr5b
4517         mov     $ctr5b, $ctr4b
4518
4519         mov     $ctr4b, $ctr3b
4520         mov     $ctr3b, $ctr1b
4521
4522         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
4523         cmp     $main_end_input_ptr, #80
4524         b.gt    .L192_dec_blocks_more_than_5
4525
4526         mov     $ctr7b, $ctr6b
4527         mov     $ctr6b, $ctr5b
4528
4529         mov     $ctr5b, $ctr4b
4530         mov     $ctr4b, $ctr1b
4531         cmp     $main_end_input_ptr, #64
4532
4533         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
4534         b.gt    .L192_dec_blocks_more_than_4
4535
4536         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
4537         mov     $ctr7b, $ctr6b
4538         mov     $ctr6b, $ctr5b
4539
4540         mov     $ctr5b, $ctr1b
4541         cmp     $main_end_input_ptr, #48
4542         b.gt    .L192_dec_blocks_more_than_3
4543
4544         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
4545         mov     $ctr7b, $ctr6b
4546         cmp     $main_end_input_ptr, #32
4547
4548         mov     $ctr6b, $ctr1b
4549         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
4550         b.gt    .L192_dec_blocks_more_than_2
4551
4552         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
4553
4554         mov     $ctr7b, $ctr1b
4555         cmp     $main_end_input_ptr, #16
4556         b.gt    .L192_dec_blocks_more_than_1
4557
4558         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
4559         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
4560         b        .L192_dec_blocks_less_than_1
4561 .L192_dec_blocks_more_than_7:                                           @ blocks left >  7
4562         rev64   $res0b, $res1b                                          @ GHASH final-7 block
4563
4564         ins     $acc_m.d[0], $h78k.d[1]                                 @ GHASH final-7 block - mid
4565         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
4566
4567         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH final-7 block - high
4568         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-7 block - mid
4569         ldr     $res1q, [$input_ptr], #16                               @ AES final-6 block - load ciphertext
4570
4571         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH final-7 block - low
4572
4573         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-7 block - mid
4574         st1     { $res4b}, [$output_ptr], #16                           @ AES final-7 block  - store result
4575
4576         eor3    $res4b, $res1b, $ctr1b, $t1.16b                         @ AES final-6 block - result
4577
4578         pmull   $acc_m.1q, $rk4v.1d, $acc_m.1d                          @ GHASH final-7 block - mid
4579         movi    $t0.8b, #0                                              @ supress further partial tag feed in
4580 .L192_dec_blocks_more_than_6:                                           @ blocks left >  6
4581
4582         rev64   $res0b, $res1b                                          @ GHASH final-6 block
4583
4584         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
4585
4586         ldr     $res1q, [$input_ptr], #16                               @ AES final-5 block - load ciphertext
4587         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-6 block - mid
4588
4589         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-6 block - mid
4590         movi    $t0.8b, #0                                              @ supress further partial tag feed in
4591         pmull2  $rk2q1, $res0.2d, $h7.2d                                @ GHASH final-6 block - high
4592
4593         st1     { $res4b}, [$output_ptr], #16                           @ AES final-6 block - store result
4594         eor3    $res4b, $res1b, $ctr2b, $t1.16b                         @ AES final-5 block - result
4595
4596         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-6 block - high
4597         pmull   $rk4v.1q, $rk4v.1d, $h78k.1d                            @ GHASH final-6 block - mid
4598         pmull   $rk3q1, $res0.1d, $h7.1d                                @ GHASH final-6 block - low
4599
4600         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-6 block - mid
4601         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-6 block - low
4602 .L192_dec_blocks_more_than_5:                                           @ blocks left >  5
4603
4604         rev64   $res0b, $res1b                                          @ GHASH final-5 block
4605
4606         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
4607
4608         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-5 block - mid
4609
4610         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-5 block - mid
4611
4612         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-5 block - mid
4613         pmull2  $rk2q1, $res0.2d, $h6.2d                                @ GHASH final-5 block - high
4614
4615         ldr     $res1q, [$input_ptr], #16                               @ AES final-4 block - load ciphertext
4616
4617         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-5 block - high
4618         pmull   $rk3q1, $res0.1d, $h6.1d                                @ GHASH final-5 block - low
4619
4620         pmull2  $rk4v.1q, $rk4v.2d, $h56k.2d                            @ GHASH final-5 block - mid
4621
4622         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-5 block - low
4623         movi    $t0.8b, #0                                              @ supress further partial tag feed in
4624         st1     { $res4b}, [$output_ptr], #16                           @ AES final-5 block - store result
4625
4626         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-5 block - mid
4627         eor3    $res4b, $res1b, $ctr3b, $t1.16b                         @ AES final-4 block - result
4628 .L192_dec_blocks_more_than_4:                                           @ blocks left >  4
4629
4630         rev64   $res0b, $res1b                                          @ GHASH final-4 block
4631
4632         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
4633         movi    $t0.8b, #0                                              @ supress further partial tag feed in
4634
4635         ldr     $res1q, [$input_ptr], #16                               @ AES final-3 block - load ciphertext
4636         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-4 block - mid
4637         pmull   $rk3q1, $res0.1d, $h5.1d                                @ GHASH final-4 block - low
4638
4639         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-4 block - mid
4640
4641         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-4 block - low
4642
4643         pmull   $rk4v.1q, $rk4v.1d, $h56k.1d                            @ GHASH final-4 block - mid
4644         st1     { $res4b}, [$output_ptr], #16                           @ AES final-4 block - store result
4645         pmull2  $rk2q1, $res0.2d, $h5.2d                                @ GHASH final-4 block - high
4646
4647         eor3    $res4b, $res1b, $ctr4b, $t1.16b                         @ AES final-3 block - result
4648
4649         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-4 block - mid
4650         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-4 block - high
4651 .L192_dec_blocks_more_than_3:                                           @ blocks left >  3
4652
4653         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
4654         ext     $h4.16b, $h4.16b, $h4.16b, #8
4655         rev64   $res0b, $res1b                                          @ GHASH final-3 block
4656         ldr     $res1q, [$input_ptr], #16                               @ AES final-2 block - load ciphertext
4657
4658         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
4659
4660         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-3 block - mid
4661         pmull2  $rk2q1, $res0.2d, $h4.2d                                @ GHASH final-3 block - high
4662
4663         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-3 block - high
4664         movi    $t0.8b, #0                                              @ supress further partial tag feed in
4665         pmull   $rk3q1, $res0.1d, $h4.1d                                @ GHASH final-3 block - low
4666
4667         st1     { $res4b}, [$output_ptr], #16                           @ AES final-3 block - store result
4668         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-3 block - mid
4669         eor3    $res4b, $res1b, $ctr5b, $t1.16b                         @ AES final-2 block - result
4670
4671         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-3 block - low
4672         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
4673
4674         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-3 block - mid
4675
4676         pmull2  $rk4v.1q, $rk4v.2d, $h34k.2d                            @ GHASH final-3 block - mid
4677
4678         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-3 block - mid
4679 .L192_dec_blocks_more_than_2:                                           @ blocks left >  2
4680
4681         rev64   $res0b, $res1b                                          @ GHASH final-2 block
4682         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
4683         ext     $h3.16b, $h3.16b, $h3.16b, #8
4684
4685         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
4686
4687         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-2 block - mid
4688         ldr     $res1q, [$input_ptr], #16                               @ AES final-1 block - load ciphertext
4689
4690         pmull2  $rk2q1, $res0.2d, $h3.2d                                @ GHASH final-2 block - high
4691
4692         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-2 block - mid
4693
4694         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-2 block - high
4695         pmull   $rk3q1, $res0.1d, $h3.1d                                @ GHASH final-2 block - low
4696
4697         pmull   $rk4v.1q, $rk4v.1d, $h34k.1d                            @ GHASH final-2 block - mid
4698         movi    $t0.8b, #0                                              @ supress further partial tag feed in
4699
4700         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-2 block - low
4701         st1     { $res4b}, [$output_ptr], #16                           @ AES final-2 block - store result
4702
4703         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-2 block - mid
4704         eor3    $res4b, $res1b, $ctr6b, $t1.16b                         @ AES final-1 block - result
4705 .L192_dec_blocks_more_than_1:                                           @ blocks left >  1
4706
4707         rev64   $res0b, $res1b                                          @ GHASH final-1 block
4708         ldr     $res1q, [$input_ptr], #16                               @ AES final block - load ciphertext
4709         ldr     $h2q, [$current_tag, #64]                               @ load h1l | h1h
4710         ext     $h2.16b, $h2.16b, $h2.16b, #8
4711
4712         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
4713         movi    $t0.8b, #0                                              @ supress further partial tag feed in
4714         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
4715
4716         pmull   $rk3q1, $res0.1d, $h2.1d                                @ GHASH final-1 block - low
4717         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-1 block - mid
4718         st1     { $res4b}, [$output_ptr], #16                           @ AES final-1 block - store result
4719
4720         pmull2  $rk2q1, $res0.2d, $h2.2d                                @ GHASH final-1 block - high
4721
4722         eor3    $res4b, $res1b, $ctr7b, $t1.16b                         @ AES final block - result
4723
4724         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-1 block - mid
4725
4726         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-1 block - mid
4727
4728         pmull2  $rk4v.1q, $rk4v.2d, $h12k.2d                            @ GHASH final-1 block - mid
4729
4730         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-1 block - low
4731
4732         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-1 block - mid
4733         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-1 block - high
4734 .L192_dec_blocks_less_than_1:                                           @ blocks left <= 1
4735
4736         rev32   $rtmp_ctr.16b, $rtmp_ctr.16b
4737         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
4738
4739         sub     $bit_length, $bit_length, #128                          @ bit_length -= 128
4740         str     $rtmp_ctrq, [$counter]                                  @ store the updated counter
4741
4742         neg     $bit_length, $bit_length                                @ bit_length = 128 - #bits in input (in range [1,128])
4743         mvn     $temp0_x, xzr                                           @ temp0_x = 0xffffffffffffffff
4744
4745         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
4746
4747         mvn     $temp1_x, xzr                                           @ temp1_x = 0xffffffffffffffff
4748         lsr     $temp0_x, $temp0_x, $bit_length                         @ temp0_x is mask for top 64b of last block
4749         cmp     $bit_length, #64
4750
4751         csel    $temp2_x, $temp1_x, $temp0_x, lt
4752         csel    $temp3_x, $temp0_x, xzr, lt
4753         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
4754         ext     $h1.16b, $h1.16b, $h1.16b, #8
4755
4756         mov     $ctr0.d[1], $temp3_x
4757         ld1     { $rk0}, [$output_ptr]                                  @ load existing bytes where the possibly partial last block is to be stored
4758
4759         mov     $ctr0.d[0], $temp2_x                                    @ ctr0b is mask for last block
4760
4761         and     $res1b, $res1b, $ctr0b                                  @ possibly partial last block has zeroes in highest bits
4762         bif     $res4b, $rk0, $ctr0b                                    @ insert existing bytes in top end of result before storing
4763
4764         rev64   $res0b, $res1b                                          @ GHASH final block
4765
4766         st1     { $res4b}, [$output_ptr]                                @ store all 16B
4767
4768         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
4769
4770         ins     $t0.d[0], $res0.d[1]                                    @ GHASH final block - mid
4771         pmull   $rk3q1, $res0.1d, $h1.1d                                @ GHASH final block - low
4772
4773         eor     $t0.8b, $t0.8b, $res0.8b                                @ GHASH final block - mid
4774         pmull2  $rk2q1, $res0.2d, $h1.2d                                @ GHASH final block - high
4775         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final block - low
4776
4777         pmull   $t0.1q, $t0.1d, $h12k.1d                                @ GHASH final block - mid
4778         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final block - high
4779
4780         eor     $t10.16b, $acc_hb, $acc_lb                              @ MODULO - karatsuba tidy up
4781         eor     $acc_mb, $acc_mb, $t0.16b                               @ GHASH final block - mid
4782         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
4783
4784         pmull   $t11.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
4785         ext     $acc_hb, $acc_hb, $acc_hb, #8                           @ MODULO - other top alignment
4786
4787         eor     $acc_mb, $acc_mb, $t10.16b                              @ MODULO - karatsuba tidy up
4788
4789         eor3    $acc_mb, $acc_mb, $acc_hb, $t11.16b                     @ MODULO - fold into mid
4790
4791         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
4792         ext     $acc_mb, $acc_mb, $acc_mb, #8                           @ MODULO - other mid alignment
4793
4794         eor3    $acc_lb, $acc_lb, $acc_mb, $acc_hb                      @ MODULO - fold into low
4795         ext     $acc_lb, $acc_lb, $acc_lb, #8
4796         rev64   $acc_lb, $acc_lb
4797         st1     { $acc_l.16b }, [$current_tag]
4798
4799         ldp     d10, d11, [sp, #16]
4800         ldp     d12, d13, [sp, #32]
4801         ldp     d14, d15, [sp, #48]
4802         ldp     d8, d9, [sp], #80
4803         ret
4804
4805 .L192_dec_ret:
4806         mov w0, #0x0
4807         ret
4808 .size unroll8_eor3_aes_gcm_dec_192_kernel,.-unroll8_eor3_aes_gcm_dec_192_kernel
4809 ___
4810 }
4811
4812 {
4813
4814 my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7));
4815 my ($temp2_x,$temp3_x)=map("x$_",(13..14));
4816 my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15));
4817 my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15));
4818 my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7));
4819 my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7));
4820 my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15));
4821
4822 my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15));
4823 my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15));
4824 my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15));
4825
4826 my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19));
4827 my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19));
4828
4829 my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25));
4830 my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25));
4831 my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25));
4832 my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25));
4833
4834 my $t0="v16";
4835 my $t0d="d16";
4836
4837 my $t1="v29";
4838 my $t2=$res1;
4839 my $t3=$t1;
4840
4841 my $t4=$res0;
4842 my $t5=$res2;
4843 my $t6=$t0;
4844
4845 my $t7=$res3;
4846 my $t8=$res4;
4847 my $t9=$res5;
4848
4849 my $t10=$res6;
4850 my $t11="v21";
4851 my $t12=$t1;
4852
4853 my $rtmp_ctr="v30";
4854 my $rtmp_ctrq="q30";
4855 my $rctr_inc="v31";
4856 my $rctr_incd="d31";
4857
4858 my $mod_constantd=$t0d;
4859 my $mod_constant=$t0;
4860
4861 my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28));
4862 my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28));
4863 my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28));
4864 my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28));
4865 my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28));
4866 my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28));
4867 my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28));
4868 my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28));
4869 my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28));
4870 my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28));
4871 my $rk2q1="v28.1q";
4872 my $rk3q1="v26.1q";
4873 my $rk4v="v27";
4874 #########################################################################################
4875 # size_t unroll8_eor3_aes_gcm_enc_256_kernel(const unsigned char *in,
4876 #                               size_t len,
4877 #                               unsigned char *out,
4878 #                               const void *key,
4879 #                               unsigned char ivec[16],
4880 #                               u64 *Xi);
4881 #
4882 $code.=<<___;
4883 .global unroll8_eor3_aes_gcm_enc_256_kernel
4884 .type   unroll8_eor3_aes_gcm_enc_256_kernel,%function
4885 .align  4
4886 unroll8_eor3_aes_gcm_enc_256_kernel:
4887         AARCH64_VALID_CALL_TARGET
4888         cbz     x1, .L256_enc_ret
4889         stp     d8, d9, [sp, #-80]!
4890         mov     $counter, x4
4891         mov     $cc, x5
4892         stp     d10, d11, [sp, #16]
4893         stp     d12, d13, [sp, #32]
4894         stp     d14, d15, [sp, #48]
4895         mov     x5, #0xc200000000000000
4896         stp     x5, xzr, [sp, #64]
4897         add     $modulo_constant, sp, #64
4898
4899         ld1     { $ctr0b}, [$counter]                                   @ CTR block 0
4900
4901         lsr     $main_end_input_ptr, $bit_length, #3                    @ byte_len
4902
4903         mov     $constant_temp, #0x100000000                    @ set up counter increment
4904         movi    $rctr_inc.16b, #0x0
4905         mov     $rctr_inc.d[1], $constant_temp
4906         sub     $main_end_input_ptr, $main_end_input_ptr, #1            @ byte_len - 1
4907
4908         and     $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80   @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
4909
4910         add     $main_end_input_ptr, $main_end_input_ptr, $input_ptr
4911
4912         rev32   $rtmp_ctr.16b, $ctr0.16b                                @ set up reversed counter
4913
4914         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 0
4915
4916         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 1
4917         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 1
4918
4919         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 2
4920         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 2
4921
4922         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 3
4923         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 3
4924
4925         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 4
4926         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 4
4927
4928         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 5
4929         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 5
4930         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
4931
4932         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 6
4933         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 6
4934
4935         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 7
4936
4937         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 0
4938         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 0
4939         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 0
4940
4941         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 0
4942         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 0
4943         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 0
4944
4945         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 0
4946         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 0
4947         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
4948
4949         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 1
4950         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 1
4951         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 1
4952
4953         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 1
4954         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 1
4955
4956         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 1
4957
4958         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 1
4959
4960         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 2
4961         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 2
4962         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 1
4963
4964         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 2
4965         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 2
4966         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 2
4967
4968         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 2
4969         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 2
4970         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 2
4971
4972         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 3
4973         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 3
4974         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
4975
4976         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 3
4977
4978         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 3
4979         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 3
4980         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 3
4981
4982         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 3
4983         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 3
4984
4985         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 4
4986         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 4
4987         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 4
4988
4989         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 4
4990         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 4
4991
4992         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 4
4993         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 4
4994         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 4
4995
4996         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 5
4997         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 5
4998         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
4999
5000         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 5
5001         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 5
5002         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 5
5003
5004         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 5
5005         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 5
5006         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 5
5007
5008         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 6
5009         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 6
5010         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 6
5011
5012         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 6
5013         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 6
5014         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 6
5015
5016         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 6
5017         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 6
5018         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
5019
5020         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 7
5021         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 7
5022
5023         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 7
5024         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 7
5025         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 7
5026
5027         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 7
5028         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 7
5029
5030         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 7
5031
5032         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 8
5033         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 8
5034
5035         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 8
5036         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 8
5037         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 8
5038
5039         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 8
5040         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 8
5041         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 8
5042
5043         ld1     { $acc_lb}, [$current_tag]
5044         ext     $acc_lb, $acc_lb, $acc_lb, #8
5045         rev64   $acc_lb, $acc_lb
5046         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
5047
5048         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 9
5049         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 9
5050         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 9
5051
5052         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 9
5053         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 9
5054         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 9
5055
5056         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 9
5057
5058         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 10
5059         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 10
5060         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 9
5061
5062         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 10
5063         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 10
5064         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 10
5065
5066         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 10
5067         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 10
5068         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 10
5069
5070         aese    $ctr4b, $rk11 \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 11
5071         ldp     $rk12q, $rk13q, [$cc, #192]                             @ load rk12, rk13
5072         aese    $ctr5b, $rk11 \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 11
5073
5074         aese    $ctr2b, $rk11 \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 11
5075         aese    $ctr6b, $rk11 \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 11
5076         aese    $ctr1b, $rk11 \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 11
5077
5078         aese    $ctr0b, $rk11 \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 11
5079         aese    $ctr3b, $rk11 \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 11
5080         aese    $ctr7b, $rk11 \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 11
5081
5082         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 7
5083         ldr     $rk14q, [$cc, #224]                                     @ load rk14
5084
5085         aese    $ctr4b, $rk12 \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 12
5086         aese    $ctr2b, $rk12 \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 12
5087         aese    $ctr1b, $rk12 \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 12
5088
5089         aese    $ctr0b, $rk12 \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 12
5090         aese    $ctr5b, $rk12 \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 12
5091         aese    $ctr3b, $rk12 \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 12
5092
5093         aese    $ctr2b, $rk13                                           @ AES block 2 - round 13
5094         aese    $ctr1b, $rk13                                           @ AES block 1 - round 13
5095         aese    $ctr4b, $rk13                                           @ AES block 4 - round 13
5096
5097         aese    $ctr6b, $rk12 \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 12
5098         aese    $ctr7b, $rk12 \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 12
5099
5100         aese    $ctr0b, $rk13                                           @ AES block 0 - round 13
5101         aese    $ctr5b, $rk13                                           @ AES block 5 - round 13
5102
5103         aese    $ctr6b, $rk13                                           @ AES block 6 - round 13
5104         aese    $ctr7b, $rk13                                           @ AES block 7 - round 13
5105         aese    $ctr3b, $rk13                                           @ AES block 3 - round 13
5106
5107         add     $end_input_ptr, $input_ptr, $bit_length, lsr #3         @ end_input_ptr
5108         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
5109         b.ge    .L256_enc_tail                                          @ handle tail
5110
5111         ldp     $ctr_t0q, $ctr_t1q, [$input_ptr], #32                   @ AES block 0, 1 - load plaintext
5112
5113         ldp     $ctr_t2q, $ctr_t3q, [$input_ptr], #32                   @ AES block 2, 3 - load plaintext
5114
5115         eor3    $res0b, $ctr_t0b, $ctr0b, $rk14                         @ AES block 0 - result
5116         rev32   $ctr0.16b, $rtmp_ctr.16b                                @ CTR block 8
5117         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8
5118
5119         eor3    $res1b, $ctr_t1b, $ctr1b, $rk14                         @ AES block 1 - result
5120         eor3    $res3b, $ctr_t3b, $ctr3b, $rk14                         @ AES block 3 - result
5121
5122         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 9
5123         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 9
5124         ldp     $ctr_t4q, $ctr_t5q, [$input_ptr], #32                   @ AES block 4, 5 - load plaintext
5125
5126         ldp     $ctr_t6q, $ctr_t7q, [$input_ptr], #32                   @ AES block 6, 7 - load plaintext
5127         eor3    $res2b, $ctr_t2b, $ctr2b, $rk14                         @ AES block 2 - result
5128         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
5129
5130         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 10
5131         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 10
5132         stp     $res0q, $res1q, [$output_ptr], #32                      @ AES block 0, 1 - store result
5133
5134         stp     $res2q, $res3q, [$output_ptr], #32                      @ AES block 2, 3 - store result
5135
5136         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 11
5137         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 11
5138
5139         eor3    $res4b, $ctr_t4b, $ctr4b, $rk14                         @ AES block 4 - result
5140
5141         eor3    $res7b, $ctr_t7b, $ctr7b, $rk14                         @ AES block 7 - result
5142         eor3    $res6b, $ctr_t6b, $ctr6b, $rk14                         @ AES block 6 - result
5143         eor3    $res5b, $ctr_t5b, $ctr5b, $rk14                         @ AES block 5 - result
5144
5145         stp     $res4q, $res5q, [$output_ptr], #32                      @ AES block 4, 5 - store result
5146         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 12
5147
5148         stp     $res6q, $res7q, [$output_ptr], #32                      @ AES block 6, 7 - store result
5149         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 12
5150         b.ge    .L256_enc_prepretail                                    @ do prepretail
5151
5152 .L256_enc_main_loop:                                                    @ main loop start
5153         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
5154
5155         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
5156         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
5157         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
5158         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
5159
5160         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
5161         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
5162         ext     $h5.16b, $h5.16b, $h5.16b, #8
5163         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
5164         ext     $h6.16b, $h6.16b, $h6.16b, #8
5165         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
5166
5167         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
5168         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
5169         rev64   $res0b, $res0b                                          @ GHASH block 8k
5170
5171         rev64   $res4b, $res4b                                          @ GHASH block 8k+4
5172         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
5173         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
5174         ext     $h7.16b, $h7.16b, $h7.16b, #8
5175         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
5176         ext     $h8.16b, $h8.16b, $h8.16b, #8
5177
5178         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
5179         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
5180         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
5181
5182         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
5183         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
5184         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
5185
5186         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
5187         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
5188         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
5189
5190         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
5191         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
5192         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
5193
5194         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
5195         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
5196         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
5197
5198         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
5199         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
5200         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
5201
5202         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
5203         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
5204         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
5205
5206         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
5207         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
5208         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
5209
5210         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
5211         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
5212         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
5213
5214         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
5215         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
5216         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
5217
5218         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
5219         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
5220         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
5221
5222         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
5223         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
5224         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
5225
5226         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
5227         rev64   $res6b, $res6b                                          @ GHASH block 8k+6
5228         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
5229
5230         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
5231         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
5232         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
5233
5234         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
5235         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
5236         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
5237
5238         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
5239         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
5240         rev64   $res5b, $res5b                                          @ GHASH block 8k+5
5241
5242         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
5243         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
5244         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
5245         ext     $h3.16b, $h3.16b, $h3.16b, #8
5246         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
5247         ext     $h4.16b, $h4.16b, $h4.16b, #8
5248
5249         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
5250         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
5251         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
5252
5253         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
5254         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
5255         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
5256
5257         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
5258         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
5259         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
5260
5261         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
5262         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
5263         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
5264
5265         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
5266         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
5267         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
5268
5269         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
5270         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
5271         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
5272
5273         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
5274         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
5275         rev64   $res7b, $res7b                                          @ GHASH block 8k+7
5276
5277         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
5278         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
5279         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
5280
5281         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
5282         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k        - mid
5283         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
5284
5285         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
5286         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
5287         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
5288
5289         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
5290         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
5291         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
5292
5293         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
5294         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
5295         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
5296
5297         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
5298         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
5299         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
5300
5301         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
5302         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
5303         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
5304
5305         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
5306         ext     $h1.16b, $h1.16b, $h1.16b, #8
5307         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
5308         ext     $h2.16b, $h2.16b, $h2.16b, #8
5309         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
5310         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
5311
5312         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
5313         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
5314         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
5315         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
5316
5317         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
5318         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
5319         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
5320
5321         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
5322         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
5323         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
5324
5325         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
5326         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
5327         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
5328
5329         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
5330         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
5331         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
5332
5333         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
5334         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 9
5335         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
5336
5337         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
5338         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
5339         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
5340
5341         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
5342         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
5343         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
5344
5345         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
5346         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
5347         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
5348
5349         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
5350         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 9
5351         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 9
5352
5353         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
5354         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 9
5355         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 9
5356
5357         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
5358         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 9
5359         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 9
5360
5361         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
5362         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
5363         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
5364
5365         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
5366         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
5367         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
5368
5369         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 9
5370
5371         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
5372         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
5373         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
5374
5375         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 10
5376         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 10
5377         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 10
5378
5379         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 10
5380         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 10
5381         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
5382
5383         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 10
5384         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 10
5385         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 10
5386
5387         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
5388
5389         ldp     $rk12q, $rk13q, [$cc, #192]                             @ load rk12, rk13
5390         rev32   $h1.16b, $rtmp_ctr.16b                                  @ CTR block 8k+16
5391
5392         ext     $t11.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
5393         ldp     $ctr_t0q, $ctr_t1q, [$input_ptr], #32                   @ AES block 8k+8, 8k+9 - load plaintext
5394         aese    $ctr2b, $rk11 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 11
5395
5396         aese    $ctr6b, $rk11 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 11
5397         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+16
5398         aese    $ctr3b, $rk11 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 11
5399
5400         aese    $ctr0b, $rk11 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 11
5401         aese    $ctr7b, $rk11 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 11
5402
5403         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
5404         aese    $ctr1b, $rk11 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 11
5405
5406         aese    $ctr7b, $rk12 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 12
5407         aese    $ctr5b, $rk11 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 11
5408
5409         aese    $ctr3b, $rk12 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 12
5410         aese    $ctr6b, $rk12 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 12
5411         rev32   $h2.16b, $rtmp_ctr.16b                                  @ CTR block 8k+17
5412
5413         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+17
5414         aese    $ctr4b, $rk11 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 11
5415         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
5416
5417         aese    $ctr5b, $rk12 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 12
5418         ldr     $rk14q, [$cc, #224]                                     @ load rk14
5419         aese    $ctr7b, $rk13                                           @ AES block 8k+15 - round 13
5420
5421         ldp     $ctr_t2q, $ctr_t3q, [$input_ptr], #32                   @ AES block 8k+10, 8k+11 - load plaintext
5422         aese    $ctr2b, $rk12 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 12
5423         aese    $ctr4b, $rk12 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 12
5424
5425         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
5426         aese    $ctr1b, $rk12 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 12
5427         ldp     $ctr_t4q, $ctr_t5q, [$input_ptr], #32                   @ AES block 4, 5 - load plaintext
5428
5429         ldp     $ctr_t6q, $ctr_t7q, [$input_ptr], #32                   @ AES block 6, 7 - load plaintext
5430         aese    $ctr2b, $rk13                                           @ AES block 8k+10 - round 13
5431         aese    $ctr4b, $rk13                                           @ AES block 8k+12 - round 13
5432
5433         rev32   $h3.16b, $rtmp_ctr.16b                                  @ CTR block 8k+18
5434         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+18
5435         aese    $ctr5b, $rk13                                           @ AES block 8k+13 - round 13
5436
5437         aese    $ctr0b, $rk12 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 12
5438         aese    $ctr3b, $rk13                                           @ AES block 8k+11 - round 13
5439         cmp     $input_ptr, $main_end_input_ptr                         @ LOOP CONTROL
5440
5441         eor3    $res2b, $ctr_t2b, $ctr2b, $rk14                         @ AES block 8k+10 - result
5442         rev32   $h4.16b, $rtmp_ctr.16b                                  @ CTR block 8k+19
5443         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+19
5444
5445         aese    $ctr0b, $rk13                                           @ AES block 8k+8 - round 13
5446         aese    $ctr6b, $rk13                                           @ AES block 8k+14 - round 13
5447         eor3    $res5b, $ctr_t5b, $ctr5b, $rk14                         @ AES block 5 - result
5448
5449         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
5450         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
5451         aese    $ctr1b, $rk13                                           @ AES block 8k+9 - round 13
5452
5453         eor3    $res4b, $ctr_t4b, $ctr4b, $rk14                         @ AES block 4 - result
5454         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 8k+20
5455         eor3    $res3b, $ctr_t3b, $ctr3b, $rk14                         @ AES block 8k+11 - result
5456
5457         mov     $ctr3.16b, $h4.16b                                      @ CTR block 8k+19
5458         eor3    $res1b, $ctr_t1b, $ctr1b, $rk14                         @ AES block 8k+9 - result
5459         eor3    $res0b, $ctr_t0b, $ctr0b, $rk14                         @ AES block 8k+8 - result
5460
5461         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+20
5462         stp     $res0q, $res1q, [$output_ptr], #32                      @ AES block 8k+8, 8k+9 - store result
5463         mov     $ctr2.16b, $h3.16b                                      @ CTR block 8k+18
5464
5465         eor3    $res7b, $ctr_t7b, $ctr7b, $rk14                         @ AES block 7 - result
5466         eor3    $acc_lb, $acc_lb, $t11.16b, $acc_hb                     @ MODULO - fold into low
5467         stp     $res2q, $res3q, [$output_ptr], #32                      @ AES block 8k+10, 8k+11 - store result
5468
5469         eor3    $res6b, $ctr_t6b, $ctr6b, $rk14                         @ AES block 6 - result
5470         mov     $ctr1.16b, $h2.16b                                      @ CTR block 8k+17
5471         stp     $res4q, $res5q, [$output_ptr], #32                      @ AES block 4, 5 - store result
5472
5473         stp     $res6q, $res7q, [$output_ptr], #32                      @ AES block 6, 7 - store result
5474         mov     $ctr0.16b, $h1.16b                                      @ CTR block 8k+16
5475         b.lt    .L256_enc_main_loop
5476
5477 .L256_enc_prepretail:                                                   @ PREPRETAIL
5478         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
5479         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
5480         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
5481
5482         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
5483
5484         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
5485         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
5486
5487         rev64   $res5b, $res5b                                          @ GHASH block 8k+5
5488         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
5489         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
5490
5491         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
5492
5493         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
5494         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
5495         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
5496
5497         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
5498         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
5499
5500         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
5501         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
5502         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
5503
5504         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
5505         rev64   $res0b, $res0b                                          @ GHASH block 8k
5506         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
5507
5508         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
5509         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
5510         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
5511
5512         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
5513         ext     $h7.16b, $h7.16b, $h7.16b, #8
5514         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
5515         ext     $h8.16b, $h8.16b, $h8.16b, #8
5516         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
5517
5518         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
5519         ext     $h5.16b, $h5.16b, $h5.16b, #8
5520         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
5521         ext     $h6.16b, $h6.16b, $h6.16b, #8
5522         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
5523         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
5524
5525         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
5526         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
5527
5528         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
5529         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
5530
5531         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
5532         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
5533         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
5534
5535         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
5536         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
5537         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
5538
5539         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
5540         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
5541         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
5542
5543         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
5544         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
5545         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
5546
5547         rev64   $res6b, $res6b                                          @ GHASH block 8k+6
5548         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
5549         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
5550
5551         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
5552         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
5553         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
5554
5555         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
5556         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
5557
5558         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
5559         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
5560         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
5561
5562         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
5563         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
5564         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
5565
5566         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
5567         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
5568         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
5569
5570         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
5571         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
5572         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
5573
5574         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
5575         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
5576         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
5577
5578         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
5579         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k        - mid
5580         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
5581
5582         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
5583         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
5584         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
5585
5586         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
5587         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
5588         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
5589
5590         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
5591         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
5592         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
5593
5594         rev64   $res4b, $res4b                                          @ GHASH block 8k+4
5595         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
5596         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
5597
5598         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
5599         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
5600         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
5601
5602         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
5603         ext     $h3.16b, $h3.16b, $h3.16b, #8
5604         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
5605         ext     $h4.16b, $h4.16b, $h4.16b, #8
5606         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
5607         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
5608
5609         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
5610         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
5611
5612         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
5613         rev64   $res7b, $res7b                                          @ GHASH block 8k+7
5614         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
5615
5616         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
5617         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
5618         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
5619
5620         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
5621         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
5622         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
5623
5624         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
5625         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
5626         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
5627         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
5628
5629         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
5630         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
5631         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
5632
5633         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
5634         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
5635         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
5636         ext     $h1.16b, $h1.16b, $h1.16b, #8
5637         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
5638         ext     $h2.16b, $h2.16b, $h2.16b, #8
5639
5640         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
5641         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
5642         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
5643
5644         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
5645         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
5646
5647         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
5648         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
5649         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
5650
5651         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
5652         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
5653         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
5654
5655         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
5656         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
5657         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
5658
5659         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
5660         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
5661         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
5662
5663         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
5664         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
5665         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
5666
5667         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
5668         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
5669         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
5670
5671         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
5672         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
5673         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
5674
5675         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
5676         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
5677         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
5678
5679         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
5680         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
5681         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
5682
5683         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
5684         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
5685         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
5686
5687         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
5688         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 9
5689         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 9
5690
5691         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
5692         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
5693         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
5694
5695         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
5696
5697         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 9
5698         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 9
5699         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 9
5700
5701         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 9
5702         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 9
5703
5704         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 10
5705         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 10
5706         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 9
5707
5708         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 10
5709         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 10
5710         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 10
5711
5712         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 10
5713         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 10
5714         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 10
5715
5716         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
5717         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
5718         aese    $ctr7b, $rk11 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 11
5719
5720         ldp     $rk12q, $rk13q, [$cc, #192]                             @ load rk12, rk13
5721         ext     $t11.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
5722         aese    $ctr2b, $rk11 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 11
5723
5724         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
5725         aese    $ctr1b, $rk11 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 11
5726         aese    $ctr6b, $rk11 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 11
5727
5728         aese    $ctr0b, $rk11 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 11
5729         aese    $ctr4b, $rk11 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 11
5730         aese    $ctr5b, $rk11 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 11
5731
5732         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
5733         aese    $ctr3b, $rk11 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 11
5734         ldr     $rk14q, [$cc, #224]                                     @ load rk14
5735
5736         aese    $ctr1b, $rk12 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 12
5737         aese    $ctr2b, $rk12 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 12
5738         aese    $ctr0b, $rk12 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 12
5739
5740         aese    $ctr6b, $rk12 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 12
5741         aese    $ctr5b, $rk12 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 12
5742         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
5743
5744         aese    $ctr4b, $rk12 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 12
5745         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
5746
5747         aese    $ctr3b, $rk12 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 12
5748         aese    $ctr7b, $rk12 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 12
5749         aese    $ctr0b, $rk13                                           @ AES block 8k+8 - round 13
5750
5751         eor3    $acc_lb, $acc_lb, $t11.16b, $acc_hb                     @ MODULO - fold into low
5752         aese    $ctr5b, $rk13                                           @ AES block 8k+13 - round 13
5753         aese    $ctr1b, $rk13                                           @ AES block 8k+9 - round 13
5754
5755         aese    $ctr3b, $rk13                                           @ AES block 8k+11 - round 13
5756         aese    $ctr4b, $rk13                                           @ AES block 8k+12 - round 13
5757         aese    $ctr7b, $rk13                                           @ AES block 8k+15 - round 13
5758
5759         aese    $ctr2b, $rk13                                           @ AES block 8k+10 - round 13
5760         aese    $ctr6b, $rk13                                           @ AES block 8k+14 - round 13
5761 .L256_enc_tail:                                                         @ TAIL
5762
5763         ldp     $h78kq, $h8q, [$current_tag, #192]                      @ load h8l | h8h
5764         ext     $h8.16b, $h8.16b, $h8.16b, #8
5765         sub     $main_end_input_ptr, $end_input_ptr, $input_ptr         @ main_end_input_ptr is number of bytes left to process
5766
5767         ldr     $ctr_t0q, [$input_ptr], #16                             @ AES block 8k+8 - load plaintext
5768
5769         ldp     $h5q, $h56kq, [$current_tag, #128]                      @ load h5l | h5h
5770         ext     $h5.16b, $h5.16b, $h5.16b, #8
5771
5772         ext     $t0.16b, $acc_lb, $acc_lb, #8                           @ prepare final partial tag
5773         ldp     $h6q, $h7q, [$current_tag, #160]                        @ load h6l | h6h
5774         ext     $h6.16b, $h6.16b, $h6.16b, #8
5775         ext     $h7.16b, $h7.16b, $h7.16b, #8
5776         mov     $t1.16b, $rk14
5777
5778         cmp     $main_end_input_ptr, #112
5779         eor3    $res1b, $ctr_t0b, $ctr0b, $t1.16b                               @ AES block 8k+8 - result
5780         b.gt    .L256_enc_blocks_more_than_7
5781
5782         movi    $acc_l.8b, #0
5783         mov     $ctr7b, $ctr6b
5784         movi    $acc_h.8b, #0
5785
5786         mov     $ctr6b, $ctr5b
5787         mov     $ctr5b, $ctr4b
5788         mov     $ctr4b, $ctr3b
5789
5790         mov     $ctr3b, $ctr2b
5791         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
5792         mov     $ctr2b, $ctr1b
5793
5794         movi    $acc_m.8b, #0
5795         cmp     $main_end_input_ptr, #96
5796         b.gt    .L256_enc_blocks_more_than_6
5797
5798         mov     $ctr7b, $ctr6b
5799         mov     $ctr6b, $ctr5b
5800         cmp     $main_end_input_ptr, #80
5801
5802         mov     $ctr5b, $ctr4b
5803         mov     $ctr4b, $ctr3b
5804         mov     $ctr3b, $ctr1b
5805
5806         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
5807         b.gt    .L256_enc_blocks_more_than_5
5808
5809         mov     $ctr7b, $ctr6b
5810         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
5811
5812         mov     $ctr6b, $ctr5b
5813         mov     $ctr5b, $ctr4b
5814
5815         cmp     $main_end_input_ptr, #64
5816         mov     $ctr4b, $ctr1b
5817         b.gt    .L256_enc_blocks_more_than_4
5818
5819         cmp     $main_end_input_ptr, #48
5820         mov     $ctr7b, $ctr6b
5821         mov     $ctr6b, $ctr5b
5822
5823         mov     $ctr5b, $ctr1b
5824         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
5825         b.gt    .L256_enc_blocks_more_than_3
5826
5827         cmp     $main_end_input_ptr, #32
5828         mov     $ctr7b, $ctr6b
5829         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
5830
5831         mov     $ctr6b, $ctr1b
5832         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
5833         b.gt    .L256_enc_blocks_more_than_2
5834
5835         mov     $ctr7b, $ctr1b
5836
5837         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
5838         cmp     $main_end_input_ptr, #16
5839         b.gt    .L256_enc_blocks_more_than_1
5840
5841         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
5842         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
5843         b        .L256_enc_blocks_less_than_1
5844 .L256_enc_blocks_more_than_7:                                           @ blocks left >  7
5845         st1     { $res1b}, [$output_ptr], #16                           @ AES final-7 block  - store result
5846
5847         rev64   $res0b, $res1b                                          @ GHASH final-7 block
5848
5849         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
5850
5851         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-6 block - load plaintext
5852
5853         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH final-7 block - high
5854         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-7 block - mid
5855         ins     $acc_m.d[0], $h78k.d[1]                                 @ GHASH final-7 block - mid
5856
5857         movi    $t0.8b, #0                                              @ supress further partial tag feed in
5858
5859         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-7 block - mid
5860         eor3    $res1b, $ctr_t1b, $ctr1b, $t1.16b                       @ AES final-6 block - result
5861
5862         pmull   $acc_m.1q, $rk4v.1d, $acc_m.1d                          @ GHASH final-7 block - mid
5863         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH final-7 block - low
5864 .L256_enc_blocks_more_than_6:                                           @ blocks left >  6
5865
5866         st1     { $res1b}, [$output_ptr], #16                           @ AES final-6 block - store result
5867
5868         rev64   $res0b, $res1b                                          @ GHASH final-6 block
5869
5870         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
5871
5872         pmull   $rk3q1, $res0.1d, $h7.1d                                @ GHASH final-6 block - low
5873         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-6 block - mid
5874         pmull2  $rk2q1, $res0.2d, $h7.2d                                @ GHASH final-6 block - high
5875
5876         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-5 block - load plaintext
5877
5878         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-6 block - low
5879
5880         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-6 block - mid
5881
5882         pmull   $rk4v.1q, $rk4v.1d, $h78k.1d                            @ GHASH final-6 block - mid
5883         eor3    $res1b, $ctr_t1b, $ctr2b, $t1.16b                       @ AES final-5 block - result
5884
5885         movi    $t0.8b, #0                                              @ supress further partial tag feed in
5886
5887         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-6 block - mid
5888         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-6 block - high
5889 .L256_enc_blocks_more_than_5:                                           @ blocks left >  5
5890
5891         st1     { $res1b}, [$output_ptr], #16                           @ AES final-5 block - store result
5892
5893         rev64   $res0b, $res1b                                          @ GHASH final-5 block
5894
5895         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
5896
5897         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-5 block - mid
5898
5899         pmull2  $rk2q1, $res0.2d, $h6.2d                                @ GHASH final-5 block - high
5900
5901         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-5 block - high
5902         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-5 block - mid
5903
5904         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-5 block - mid
5905
5906         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-4 block - load plaintext
5907         pmull   $rk3q1, $res0.1d, $h6.1d                                @ GHASH final-5 block - low
5908
5909         pmull2  $rk4v.1q, $rk4v.2d, $h56k.2d                            @ GHASH final-5 block - mid
5910         movi    $t0.8b, #0                                              @ supress further partial tag feed in
5911         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-5 block - low
5912
5913         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-5 block - mid
5914         eor3    $res1b, $ctr_t1b, $ctr3b, $t1.16b                       @ AES final-4 block - result
5915 .L256_enc_blocks_more_than_4:                                           @ blocks left >  4
5916
5917         st1     { $res1b}, [$output_ptr], #16                           @ AES final-4 block - store result
5918
5919         rev64   $res0b, $res1b                                          @ GHASH final-4 block
5920
5921         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-3 block - load plaintext
5922
5923         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
5924
5925         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-4 block - mid
5926         pmull2  $rk2q1, $res0.2d, $h5.2d                                @ GHASH final-4 block - high
5927
5928         eor3    $res1b, $ctr_t1b, $ctr4b, $t1.16b                       @ AES final-3 block - result
5929         pmull   $rk3q1, $res0.1d, $h5.1d                                @ GHASH final-4 block - low
5930
5931         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-4 block - mid
5932         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-4 block - low
5933
5934         pmull   $rk4v.1q, $rk4v.1d, $h56k.1d                            @ GHASH final-4 block - mid
5935
5936         movi    $t0.8b, #0                                              @ supress further partial tag feed in
5937
5938         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-4 block - mid
5939         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-4 block - high
5940 .L256_enc_blocks_more_than_3:                                           @ blocks left >  3
5941
5942         st1     { $res1b}, [$output_ptr], #16                           @ AES final-3 block - store result
5943
5944         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
5945         ext     $h4.16b, $h4.16b, $h4.16b, #8
5946         rev64   $res0b, $res1b                                          @ GHASH final-3 block
5947
5948         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
5949
5950         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-3 block - mid
5951         pmull2  $rk2q1, $res0.2d, $h4.2d                                @ GHASH final-3 block - high
5952
5953         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-3 block - high
5954         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-3 block - mid
5955         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
5956
5957         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-3 block - mid
5958         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-2 block - load plaintext
5959
5960         pmull2  $rk4v.1q, $rk4v.2d, $h34k.2d                            @ GHASH final-3 block - mid
5961         pmull   $rk3q1, $res0.1d, $h4.1d                                @ GHASH final-3 block - low
5962
5963         eor3    $res1b, $ctr_t1b, $ctr5b, $t1.16b                       @ AES final-2 block - result
5964         movi    $t0.8b, #0                                              @ supress further partial tag feed in
5965
5966         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-3 block - mid
5967         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-3 block - low
5968 .L256_enc_blocks_more_than_2:                                           @ blocks left >  2
5969
5970         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
5971         ext     $h3.16b, $h3.16b, $h3.16b, #8
5972
5973         st1     { $res1b}, [$output_ptr], #16                           @ AES final-2 block - store result
5974
5975         rev64   $res0b, $res1b                                          @ GHASH final-2 block
5976         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final-1 block - load plaintext
5977
5978         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
5979
5980         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-2 block - mid
5981
5982         movi    $t0.8b, #0                                              @ supress further partial tag feed in
5983
5984         pmull2  $rk2q1, $res0.2d, $h3.2d                                @ GHASH final-2 block - high
5985         eor3    $res1b, $ctr_t1b, $ctr6b, $t1.16b                       @ AES final-1 block - result
5986
5987         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-2 block - mid
5988
5989         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-2 block - high
5990
5991         pmull   $rk4v.1q, $rk4v.1d, $h34k.1d                            @ GHASH final-2 block - mid
5992         pmull   $rk3q1, $res0.1d, $h3.1d                                @ GHASH final-2 block - low
5993
5994         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-2 block - mid
5995         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-2 block - low
5996 .L256_enc_blocks_more_than_1:                                           @ blocks left >  1
5997
5998         st1     { $res1b}, [$output_ptr], #16                           @ AES final-1 block - store result
5999
6000         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
6001         ext     $h2.16b, $h2.16b, $h2.16b, #8
6002         rev64   $res0b, $res1b                                          @ GHASH final-1 block
6003         ldr     $ctr_t1q, [$input_ptr], #16                             @ AES final block - load plaintext
6004
6005         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
6006         movi    $t0.8b, #0                                              @ supress further partial tag feed in
6007
6008         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-1 block - mid
6009         pmull2  $rk2q1, $res0.2d, $h2.2d                                @ GHASH final-1 block - high
6010
6011         eor3    $res1b, $ctr_t1b, $ctr7b, $t1.16b                       @ AES final block - result
6012         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-1 block - high
6013
6014         pmull   $rk3q1, $res0.1d, $h2.1d                                @ GHASH final-1 block - low
6015         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-1 block - mid
6016
6017         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
6018
6019         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-1 block - low
6020         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-1 block - mid
6021
6022         pmull2  $rk4v.1q, $rk4v.2d, $h12k.2d                            @ GHASH final-1 block - mid
6023
6024         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-1 block - mid
6025 .L256_enc_blocks_less_than_1:                                           @ blocks left <= 1
6026
6027         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
6028
6029         sub     $bit_length, $bit_length, #128                          @ bit_length -= 128
6030
6031         neg     $bit_length, $bit_length                                @ bit_length = 128 - #bits in input (in range [1,128])
6032
6033         mvn     $temp0_x, xzr                                           @ temp0_x = 0xffffffffffffffff
6034         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
6035
6036         lsr     $temp0_x, $temp0_x, $bit_length                         @ temp0_x is mask for top 64b of last block
6037         cmp     $bit_length, #64
6038         mvn     $temp1_x, xzr                                           @ temp1_x = 0xffffffffffffffff
6039
6040         csel    $temp3_x, $temp0_x, xzr, lt
6041         csel    $temp2_x, $temp1_x, $temp0_x, lt
6042
6043         mov     $ctr0.d[0], $temp2_x                                    @ ctr0b is mask for last block
6044         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
6045         ext     $h1.16b, $h1.16b, $h1.16b, #8
6046
6047         ld1     { $rk0}, [$output_ptr]                                  @ load existing bytes where the possibly partial last block is to be stored
6048         mov     $ctr0.d[1], $temp3_x
6049
6050         and     $res1b, $res1b, $ctr0b                                  @ possibly partial last block has zeroes in highest bits
6051
6052         rev64   $res0b, $res1b                                          @ GHASH final block
6053
6054         rev32   $rtmp_ctr.16b, $rtmp_ctr.16b
6055         bif     $res1b, $rk0, $ctr0b                                    @ insert existing bytes in top end of result before storing
6056         str     $rtmp_ctrq, [$counter]                                  @ store the updated counter
6057
6058         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
6059         st1     { $res1b}, [$output_ptr]                                @ store all 16B
6060
6061         ins     $t0.d[0], $res0.d[1]                                    @ GHASH final block - mid
6062         pmull2  $rk2q1, $res0.2d, $h1.2d                                @ GHASH final block - high
6063         pmull   $rk3q1, $res0.1d, $h1.1d                                @ GHASH final block - low
6064
6065         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final block - high
6066         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final block - low
6067
6068         eor     $t0.8b, $t0.8b, $res0.8b                                @ GHASH final block - mid
6069
6070         pmull   $t0.1q, $t0.1d, $h12k.1d                                @ GHASH final block - mid
6071
6072         eor     $acc_mb, $acc_mb, $t0.16b                               @ GHASH final block - mid
6073         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
6074
6075         ext     $t11.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
6076
6077         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
6078         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
6079
6080         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
6081
6082         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
6083         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
6084
6085         eor3    $acc_lb, $acc_lb, $acc_hb, $t11.16b                     @ MODULO - fold into low
6086                 ext     $acc_lb, $acc_lb, $acc_lb, #8
6087         rev64   $acc_lb, $acc_lb
6088         st1     { $acc_l.16b }, [$current_tag]
6089         lsr     x0, $bit_length, #3                                     @ return sizes
6090
6091         ldp     d10, d11, [sp, #16]
6092         ldp     d12, d13, [sp, #32]
6093         ldp     d14, d15, [sp, #48]
6094         ldp     d8, d9, [sp], #80
6095         ret
6096
6097 .L256_enc_ret:
6098         mov w0, #0x0
6099         ret
6100 .size unroll8_eor3_aes_gcm_enc_256_kernel,.-unroll8_eor3_aes_gcm_enc_256_kernel
6101 ___
6102
6103 {
6104 #########################################################################################
6105 # size_t unroll8_eor3_aes_gcm_dec_256_kernel(const unsigned char *in,
6106 #                               size_t len,
6107 #                               unsigned char *out,
6108 #                               const void *key,
6109 #                               unsigned char ivec[16],
6110 #                               u64 *Xi);
6111 #
6112 $code.=<<___;
6113 .global unroll8_eor3_aes_gcm_dec_256_kernel
6114 .type   unroll8_eor3_aes_gcm_dec_256_kernel,%function
6115 .align  4
6116 unroll8_eor3_aes_gcm_dec_256_kernel:
6117         AARCH64_VALID_CALL_TARGET
6118         cbz     x1, .L256_dec_ret
6119         stp     d8, d9, [sp, #-80]!
6120         mov     $counter, x4
6121         mov     $cc, x5
6122         stp     d10, d11, [sp, #16]
6123         stp     d12, d13, [sp, #32]
6124         stp     d14, d15, [sp, #48]
6125         mov     x5, #0xc200000000000000
6126         stp     x5, xzr, [sp, #64]
6127         add     $modulo_constant, sp, #64
6128
6129         ld1     { $ctr0b}, [$counter]                                   @ CTR block 0
6130
6131         mov     $constant_temp, #0x100000000                    @ set up counter increment
6132         movi    $rctr_inc.16b, #0x0
6133         mov     $rctr_inc.d[1], $constant_temp
6134         lsr     $main_end_input_ptr, $bit_length, #3                    @ byte_len
6135
6136         sub     $main_end_input_ptr, $main_end_input_ptr, #1            @ byte_len - 1
6137
6138         rev32   $rtmp_ctr.16b, $ctr0.16b                                @ set up reversed counter
6139
6140         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 0
6141
6142         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 1
6143         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 1
6144
6145         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 2
6146         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 2
6147         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
6148
6149         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 3
6150         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 3
6151
6152         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 4
6153         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 4
6154
6155         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 0
6156
6157         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 5
6158         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 5
6159
6160         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 0
6161         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 0
6162
6163         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 6
6164         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 6
6165
6166         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 7
6167         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 0
6168
6169         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 0
6170         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 0
6171
6172         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 0
6173         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 0
6174         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
6175
6176         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 1
6177         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 1
6178         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 1
6179
6180         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 1
6181         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 1
6182         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 1
6183
6184         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 1
6185         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 1
6186
6187         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 2
6188         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 2
6189         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 2
6190
6191         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 2
6192         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 2
6193         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 2
6194
6195         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 2
6196         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 2
6197         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
6198
6199         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 3
6200         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 3
6201
6202         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 3
6203         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 3
6204
6205         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 3
6206         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 3
6207         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 3
6208
6209         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 3
6210
6211         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 4
6212         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 4
6213
6214         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 4
6215         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 4
6216         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 4
6217
6218         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 4
6219         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 4
6220         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 4
6221
6222         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 5
6223         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 5
6224
6225         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
6226         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 5
6227         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 5
6228
6229         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 5
6230
6231         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 5
6232         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 5
6233
6234         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 5
6235
6236         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 6
6237         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 6
6238         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 6
6239
6240         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 6
6241         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 6
6242         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 6
6243
6244         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 6
6245         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 6
6246         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
6247
6248         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 7
6249         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 7
6250
6251         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 7
6252         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 7
6253         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 7
6254
6255         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 7
6256         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 7
6257         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 7
6258
6259         and     $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
6260         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 8
6261         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 8
6262
6263         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 8
6264         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 8
6265         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 8
6266
6267         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 8
6268         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 8
6269         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 8
6270
6271         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 9
6272
6273         ld1     { $acc_lb}, [$current_tag]
6274         ext     $acc_lb, $acc_lb, $acc_lb, #8
6275         rev64   $acc_lb, $acc_lb
6276         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
6277         add     $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr
6278         add     $main_end_input_ptr, $main_end_input_ptr, $input_ptr
6279
6280         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 9
6281         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 9
6282
6283         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 9
6284         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 9
6285
6286         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 9
6287
6288         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 9
6289         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 9
6290
6291         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 10
6292         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 10
6293         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 10
6294
6295         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 10
6296         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 10
6297         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 10
6298
6299         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 10
6300         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 10
6301         ldp     $rk12q, $rk13q, [$cc, #192]                             @ load rk12, rk13
6302
6303         aese    $ctr0b, $rk11 \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 11
6304         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7
6305
6306         aese    $ctr7b, $rk11 \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 11
6307         aese    $ctr3b, $rk11 \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 11
6308         aese    $ctr1b, $rk11 \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 11
6309
6310         aese    $ctr5b, $rk11 \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 11
6311         aese    $ctr4b, $rk11 \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 11
6312         aese    $ctr2b, $rk11 \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 11
6313
6314         aese    $ctr6b, $rk11 \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 11
6315         ldr     $rk14q, [$cc, #224]                                     @ load rk14
6316
6317         aese    $ctr1b, $rk12 \n  aesmc $ctr1b, $ctr1b                  @ AES block 1 - round 12
6318         aese    $ctr4b, $rk12 \n  aesmc $ctr4b, $ctr4b                  @ AES block 4 - round 12
6319         aese    $ctr5b, $rk12 \n  aesmc $ctr5b, $ctr5b                  @ AES block 5 - round 12
6320
6321         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
6322         aese    $ctr3b, $rk12 \n  aesmc $ctr3b, $ctr3b                  @ AES block 3 - round 12
6323         aese    $ctr2b, $rk12 \n  aesmc $ctr2b, $ctr2b                  @ AES block 2 - round 12
6324
6325         aese    $ctr6b, $rk12 \n  aesmc $ctr6b, $ctr6b                  @ AES block 6 - round 12
6326         aese    $ctr0b, $rk12 \n  aesmc $ctr0b, $ctr0b                  @ AES block 0 - round 12
6327         aese    $ctr7b, $rk12 \n  aesmc $ctr7b, $ctr7b                  @ AES block 7 - round 12
6328
6329         aese    $ctr5b, $rk13                                           @ AES block 5 - round 13
6330         aese    $ctr1b, $rk13                                           @ AES block 1 - round 13
6331         aese    $ctr2b, $rk13                                           @ AES block 2 - round 13
6332
6333         aese    $ctr0b, $rk13                                           @ AES block 0 - round 13
6334         aese    $ctr4b, $rk13                                           @ AES block 4 - round 13
6335         aese    $ctr6b, $rk13                                           @ AES block 6 - round 13
6336
6337         aese    $ctr3b, $rk13                                           @ AES block 3 - round 13
6338         aese    $ctr7b, $rk13                                           @ AES block 7 - round 13
6339         b.ge    .L256_dec_tail                                          @ handle tail
6340
6341         ldp     $res0q, $res1q, [$input_ptr], #32                       @ AES block 0, 1 - load ciphertext
6342
6343         ldp     $res2q, $res3q, [$input_ptr], #32                       @ AES block 2, 3 - load ciphertext
6344
6345         ldp     $res4q, $res5q, [$input_ptr], #32                       @ AES block 4, 5 - load ciphertext
6346
6347         ldp     $res6q, $res7q, [$input_ptr], #32                       @ AES block 6, 7 - load ciphertext
6348         cmp     $input_ptr, $main_end_input_ptr                         @ check if we have <= 8 blocks
6349
6350         eor3    $ctr1b, $res1b, $ctr1b, $rk14                           @ AES block 1 - result
6351         eor3    $ctr0b, $res0b, $ctr0b, $rk14                           @ AES block 0 - result
6352         stp     $ctr0q, $ctr1q, [$output_ptr], #32                      @ AES block 0, 1 - store result
6353
6354         rev32   $ctr0.16b, $rtmp_ctr.16b                                @ CTR block 8
6355         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8
6356         eor3    $ctr3b, $res3b, $ctr3b, $rk14                           @ AES block 3 - result
6357
6358         eor3    $ctr5b, $res5b, $ctr5b, $rk14                           @ AES block 5 - result
6359
6360         eor3    $ctr4b, $res4b, $ctr4b, $rk14                           @ AES block 4 - result
6361         rev32   $ctr1.16b, $rtmp_ctr.16b                                @ CTR block 9
6362         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 9
6363
6364         eor3    $ctr2b, $res2b, $ctr2b, $rk14                           @ AES block 2 - result
6365         stp     $ctr2q, $ctr3q, [$output_ptr], #32                      @ AES block 2, 3 - store result
6366
6367         rev32   $ctr2.16b, $rtmp_ctr.16b                                @ CTR block 10
6368         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 10
6369
6370         eor3    $ctr6b, $res6b, $ctr6b, $rk14                           @ AES block 6 - result
6371
6372         rev32   $ctr3.16b, $rtmp_ctr.16b                                @ CTR block 11
6373         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 11
6374         stp     $ctr4q, $ctr5q, [$output_ptr], #32                      @ AES block 4, 5 - store result
6375
6376         eor3    $ctr7b, $res7b, $ctr7b, $rk14                           @ AES block 7 - result
6377         stp     $ctr6q, $ctr7q, [$output_ptr], #32                      @ AES block 6, 7 - store result
6378
6379         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 12
6380         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 12
6381         b.ge    .L256_dec_prepretail                                    @ do prepretail
6382
6383 .L256_dec_main_loop:                                                    @ main loop start
6384         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
6385         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
6386         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
6387
6388         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
6389         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
6390         ext     $h7.16b, $h7.16b, $h7.16b, #8
6391         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
6392         ext     $h8.16b, $h8.16b, $h8.16b, #8
6393
6394         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
6395         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
6396         rev64   $res0b, $res0b                                          @ GHASH block 8k
6397
6398         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
6399         rev64   $res4b, $res4b                                          @ GHASH block 8k+4
6400         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
6401
6402         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
6403         rev64   $res7b, $res7b                                          @ GHASH block 8k+7
6404
6405         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
6406         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
6407         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
6408
6409         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
6410         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
6411         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
6412
6413         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
6414         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
6415         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
6416
6417         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
6418         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
6419         ext     $h5.16b, $h5.16b, $h5.16b, #8
6420         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
6421         ext     $h6.16b, $h6.16b, $h6.16b, #8
6422         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
6423
6424         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
6425         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
6426         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
6427
6428         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
6429         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
6430         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
6431
6432         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
6433         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
6434         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
6435
6436         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
6437         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
6438         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
6439
6440         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
6441         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
6442         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
6443
6444         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
6445         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
6446         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
6447
6448         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
6449         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
6450         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
6451
6452         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
6453         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
6454         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
6455
6456         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
6457         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
6458         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
6459
6460         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
6461         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
6462         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
6463
6464         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
6465         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
6466         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
6467
6468         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
6469         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
6470         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
6471
6472         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
6473         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
6474         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
6475
6476         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
6477         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
6478         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
6479
6480         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
6481         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
6482         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
6483         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
6484
6485         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
6486         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
6487         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
6488
6489         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
6490         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
6491         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
6492
6493         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
6494         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
6495         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
6496
6497         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
6498         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
6499         rev64   $res5b, $res5b                                          @ GHASH block 8k+5
6500
6501         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k        - mid
6502         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
6503         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
6504
6505         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
6506         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
6507         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
6508
6509         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
6510         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
6511         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
6512
6513         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
6514         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
6515         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
6516
6517         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
6518         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
6519         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
6520
6521         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
6522         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
6523         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
6524
6525         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
6526         ext     $h3.16b, $h3.16b, $h3.16b, #8
6527         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
6528         ext     $h4.16b, $h4.16b, $h4.16b, #8
6529         rev64   $res6b, $res6b                                          @ GHASH block 8k+6
6530         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
6531
6532         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
6533         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
6534         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
6535
6536         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
6537         ext     $h1.16b, $h1.16b, $h1.16b, #8
6538         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
6539         ext     $h2.16b, $h2.16b, $h2.16b, #8
6540         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
6541         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
6542
6543         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
6544         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
6545         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
6546
6547         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
6548         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
6549         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
6550         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
6551
6552         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
6553         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
6554         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
6555
6556         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
6557         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
6558         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
6559
6560         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
6561         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
6562         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
6563
6564         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
6565         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
6566         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
6567
6568         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
6569         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
6570         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
6571
6572         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
6573         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
6574         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
6575
6576         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
6577         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
6578         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 9
6579
6580         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 9
6581         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
6582         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 9
6583
6584         ldp     $res0q, $res1q, [$input_ptr], #32                       @ AES block 8k+8, 8k+9 - load ciphertext
6585         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
6586         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 9
6587
6588         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
6589         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 9
6590         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 9
6591
6592         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
6593         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
6594         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
6595
6596         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
6597         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 10
6598         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 10
6599
6600         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
6601         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 9
6602         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
6603
6604         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 9
6605         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
6606         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
6607
6608         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 10
6609         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 10
6610         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 10
6611
6612         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 10
6613         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 10
6614         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 10
6615
6616         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
6617         rev32   $h1.16b, $rtmp_ctr.16b                                  @ CTR block 8k+16
6618         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
6619
6620         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+16
6621         aese    $ctr1b, $rk11 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 11
6622         ldp     $rk12q, $rk13q, [$cc, #192]                             @ load rk12, rk13
6623
6624         aese    $ctr0b, $rk11 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 11
6625         aese    $ctr6b, $rk11 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 11
6626
6627         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
6628         rev32   $h2.16b, $rtmp_ctr.16b                                  @ CTR block 8k+17
6629         aese    $ctr2b, $rk11 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 11
6630
6631         ldp     $res2q, $res3q, [$input_ptr], #32                       @ AES block 8k+10, 8k+11 - load ciphertext
6632         aese    $ctr7b, $rk11 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 11
6633         ext     $t11.16b, $acc_hb, $acc_hb, #8                           @ MODULO - other top alignment
6634
6635         aese    $ctr5b, $rk11 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 11
6636         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+17
6637         aese    $ctr3b, $rk11 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 11
6638
6639         aese    $ctr2b, $rk12 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 12
6640         aese    $ctr7b, $rk12 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 12
6641         aese    $ctr6b, $rk12 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 12
6642
6643         rev32   $h3.16b, $rtmp_ctr.16b                                  @ CTR block 8k+18
6644         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+18
6645         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
6646
6647         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
6648         aese    $ctr1b, $rk12 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 12
6649         aese    $ctr4b, $rk11 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 11
6650
6651         ldr     $rk14q, [$cc, #224]                                     @ load rk14
6652         aese    $ctr5b, $rk12 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 12
6653         aese    $ctr3b, $rk12 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 12
6654
6655         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
6656         aese    $ctr0b, $rk12 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 12
6657         aese    $ctr4b, $rk12 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 12
6658
6659         ldp     $res4q, $res5q, [$input_ptr], #32                       @ AES block 8k+12, 8k+13 - load ciphertext
6660         aese    $ctr1b, $rk13                                           @ AES block 8k+9 - round 13
6661         aese    $ctr2b, $rk13                                           @ AES block 8k+10 - round 13
6662
6663         ldp     $res6q, $res7q, [$input_ptr], #32                       @ AES block 8k+14, 8k+15 - load ciphertext
6664         aese    $ctr0b, $rk13                                           @ AES block 8k+8 - round 13
6665         aese    $ctr5b, $rk13                                           @ AES block 8k+13 - round 13
6666
6667         rev32   $h4.16b, $rtmp_ctr.16b                                  @ CTR block 8k+19
6668         eor3    $ctr2b, $res2b, $ctr2b, $rk14                           @ AES block 8k+10 - result
6669         eor3    $ctr1b, $res1b, $ctr1b, $rk14                           @ AES block 8k+9 - result
6670
6671         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
6672         aese    $ctr7b, $rk13                                           @ AES block 8k+15 - round 13
6673
6674         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+19
6675         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
6676         aese    $ctr4b, $rk13                                           @ AES block 8k+12 - round 13
6677
6678         eor3    $ctr5b, $res5b, $ctr5b, $rk14                           @ AES block 8k+13 - result
6679         eor3    $ctr0b, $res0b, $ctr0b, $rk14                           @ AES block 8k+8 - result
6680         aese    $ctr3b, $rk13                                           @ AES block 8k+11 - round 13
6681
6682         stp     $ctr0q, $ctr1q, [$output_ptr], #32                      @ AES block 8k+8, 8k+9 - store result
6683         mov     $ctr0.16b, $h1.16b                                      @ CTR block 8k+16
6684         eor3    $ctr4b, $res4b, $ctr4b, $rk14                           @ AES block 8k+12 - result
6685
6686         eor3    $acc_lb, $acc_lb, $t11.16b, $acc_hb                     @ MODULO - fold into low
6687         eor3    $ctr3b, $res3b, $ctr3b, $rk14                           @ AES block 8k+11 - result
6688         stp     $ctr2q, $ctr3q, [$output_ptr], #32                      @ AES block 8k+10, 8k+11 - store result
6689
6690         mov     $ctr3.16b, $h4.16b                                      @ CTR block 8k+19
6691         mov     $ctr2.16b, $h3.16b                                      @ CTR block 8k+18
6692         aese    $ctr6b, $rk13                                           @ AES block 8k+14 - round 13
6693
6694         mov     $ctr1.16b, $h2.16b                                      @ CTR block 8k+17
6695         stp     $ctr4q, $ctr5q, [$output_ptr], #32                      @ AES block 8k+12, 8k+13 - store result
6696         eor3    $ctr7b, $res7b, $ctr7b, $rk14                           @ AES block 8k+15 - result
6697
6698         eor3    $ctr6b, $res6b, $ctr6b, $rk14                           @ AES block 8k+14 - result
6699         rev32   $ctr4.16b, $rtmp_ctr.16b                                @ CTR block 8k+20
6700         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+20
6701
6702         cmp     $input_ptr, $main_end_input_ptr                         @ LOOP CONTROL
6703         stp     $ctr6q, $ctr7q, [$output_ptr], #32                      @ AES block 8k+14, 8k+15 - store result
6704         b.lt    .L256_dec_main_loop
6705
6706 .L256_dec_prepretail:                                                   @ PREPRETAIL
6707         ldp     $rk0q, $rk1q, [$cc, #0]                                 @ load rk0, rk1
6708         rev32   $ctr5.16b, $rtmp_ctr.16b                                @ CTR block 8k+13
6709         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+13
6710
6711         rev64   $res4b, $res4b                                          @ GHASH block 8k+4
6712         ldr     $h56kq, [$current_tag, #144]                            @ load h6k | h5k
6713         ldr     $h78kq, [$current_tag, #192]                            @ load h8k | h7k
6714
6715         rev32   $ctr6.16b, $rtmp_ctr.16b                                @ CTR block 8k+14
6716         rev64   $res0b, $res0b                                          @ GHASH block 8k
6717         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+14
6718
6719         ext     $acc_lb, $acc_lb, $acc_lb, #8                           @ PRE 0
6720         ldr     $h7q, [$current_tag, #176]                              @ load h7l | h7h
6721         ext     $h7.16b, $h7.16b, $h7.16b, #8
6722         ldr     $h8q, [$current_tag, #208]                              @ load h8l | h8h
6723         ext     $h8.16b, $h8.16b, $h8.16b, #8
6724         rev64   $res1b, $res1b                                          @ GHASH block 8k+1
6725
6726         rev32   $ctr7.16b, $rtmp_ctr.16b                                @ CTR block 8k+15
6727         rev64   $res2b, $res2b                                          @ GHASH block 8k+2
6728         ldr     $h5q, [$current_tag, #128]                              @ load h5l | h5h
6729         ext     $h5.16b, $h5.16b, $h5.16b, #8
6730         ldr     $h6q, [$current_tag, #160]                              @ load h6l | h6h
6731         ext     $h6.16b, $h6.16b, $h6.16b, #8
6732
6733         aese    $ctr0b, $rk0  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 0
6734         aese    $ctr1b, $rk0  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 0
6735         aese    $ctr4b, $rk0  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 0
6736
6737         aese    $ctr3b, $rk0  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 0
6738         aese    $ctr5b, $rk0  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 0
6739         aese    $ctr6b, $rk0  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 0
6740
6741         aese    $ctr4b, $rk1  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 1
6742         aese    $ctr7b, $rk0  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 0
6743         aese    $ctr2b, $rk0  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 0
6744
6745         ldp     $rk2q, $rk3q, [$cc, #32]                                @ load rk2, rk3
6746         aese    $ctr0b, $rk1  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 1
6747         eor     $res0b, $res0b, $acc_lb                                 @ PRE 1
6748
6749         aese    $ctr7b, $rk1  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 1
6750         aese    $ctr6b, $rk1  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 1
6751         aese    $ctr2b, $rk1  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 1
6752
6753         aese    $ctr3b, $rk1  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 1
6754         aese    $ctr1b, $rk1  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 1
6755         aese    $ctr5b, $rk1  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 1
6756
6757         pmull2  $t0.1q, $res1.2d, $h7.2d                                @ GHASH block 8k+1 - high
6758         trn1    $acc_m.2d, $res1.2d, $res0.2d                           @ GHASH block 8k, 8k+1 - mid
6759         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH block 8k - low
6760
6761         rev64   $res3b, $res3b                                          @ GHASH block 8k+3
6762         pmull   $h7.1q, $res1.1d, $h7.1d                                @ GHASH block 8k+1 - low
6763
6764         aese    $ctr5b, $rk2  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 2
6765         aese    $ctr7b, $rk2  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 2
6766         aese    $ctr1b, $rk2  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 2
6767
6768         aese    $ctr3b, $rk2  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 2
6769         aese    $ctr6b, $rk2  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 2
6770         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH block 8k - high
6771
6772         aese    $ctr0b, $rk2  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 2
6773         aese    $ctr7b, $rk3  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 3
6774
6775         aese    $ctr5b, $rk3  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 3
6776         rev64   $res6b, $res6b                                          @ GHASH block 8k+6
6777
6778         aese    $ctr0b, $rk3  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 3
6779         aese    $ctr2b, $rk2  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 2
6780         aese    $ctr6b, $rk3  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 3
6781
6782         pmull2  $t1.1q, $res2.2d, $h6.2d                                @ GHASH block 8k+2 - high
6783         trn2    $res0.2d, $res1.2d, $res0.2d                            @ GHASH block 8k, 8k+1 - mid
6784         aese    $ctr4b, $rk2  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 2
6785
6786         ldp     $rk4q, $rk5q, [$cc, #64]                                @ load rk4, rk5
6787         aese    $ctr1b, $rk3  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 3
6788         pmull2  $t2.1q, $res3.2d, $h5.2d                                @ GHASH block 8k+3 - high
6789
6790         aese    $ctr2b, $rk3  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 3
6791         eor     $acc_hb, $acc_hb, $t0.16b                               @ GHASH block 8k+1 - high
6792         eor     $res0.16b, $res0.16b, $acc_m.16b                        @ GHASH block 8k, 8k+1 - mid
6793
6794         aese    $ctr4b, $rk3  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 3
6795         pmull   $h6.1q, $res2.1d, $h6.1d                                @ GHASH block 8k+2 - low
6796         aese    $ctr3b, $rk3  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 3
6797
6798         eor3    $acc_hb, $acc_hb, $t1.16b, $t2.16b                      @ GHASH block 8k+2, 8k+3 - high
6799         trn1    $t3.2d, $res3.2d, $res2.2d                              @ GHASH block 8k+2, 8k+3 - mid
6800         trn2    $res2.2d, $res3.2d, $res2.2d                            @ GHASH block 8k+2, 8k+3 - mid
6801
6802         pmull2  $acc_m.1q, $res0.2d, $h78k.2d                           @ GHASH block 8k        - mid
6803         pmull   $h5.1q, $res3.1d, $h5.1d                                @ GHASH block 8k+3 - low
6804         eor     $acc_lb, $acc_lb, $h7.16b                               @ GHASH block 8k+1 - low
6805
6806         pmull   $h78k.1q, $res0.1d, $h78k.1d                            @ GHASH block 8k+1 - mid
6807         aese    $ctr5b, $rk4  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 4
6808         aese    $ctr0b, $rk4  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 4
6809
6810         eor3    $acc_lb, $acc_lb, $h6.16b, $h5.16b                      @ GHASH block 8k+2, 8k+3 - low
6811         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
6812         ext     $h1.16b, $h1.16b, $h1.16b, #8
6813         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
6814         ext     $h2.16b, $h2.16b, $h2.16b, #8
6815         aese    $ctr7b, $rk4  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 4
6816
6817         aese    $ctr2b, $rk4  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 4
6818         aese    $ctr6b, $rk4  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 4
6819         eor     $acc_mb, $acc_mb, $h78k.16b                             @ GHASH block 8k+1 - mid
6820
6821         eor     $res2.16b, $res2.16b, $t3.16b                           @ GHASH block 8k+2, 8k+3 - mid
6822         aese    $ctr7b, $rk5  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 5
6823         aese    $ctr1b, $rk4  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 4
6824
6825         aese    $ctr2b, $rk5  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 5
6826         aese    $ctr3b, $rk4  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 4
6827         aese    $ctr4b, $rk4  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 4
6828
6829         aese    $ctr1b, $rk5  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 5
6830         pmull2  $t3.1q, $res2.2d, $h56k.2d                              @ GHASH block 8k+2 - mid
6831         aese    $ctr6b, $rk5  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 5
6832
6833         aese    $ctr4b, $rk5  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 5
6834         aese    $ctr3b, $rk5  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 5
6835         pmull   $h56k.1q, $res2.1d, $h56k.1d                            @ GHASH block 8k+3 - mid
6836
6837         aese    $ctr0b, $rk5  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 5
6838         aese    $ctr5b, $rk5  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 5
6839         ldp     $rk6q, $rk7q, [$cc, #96]                                @ load rk6, rk7
6840
6841         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
6842         ext     $h3.16b, $h3.16b, $h3.16b, #8
6843         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
6844         ext     $h4.16b, $h4.16b, $h4.16b, #8
6845         rev64   $res7b, $res7b                                          @ GHASH block 8k+7
6846         rev64   $res5b, $res5b                                          @ GHASH block 8k+5
6847
6848         eor3    $acc_mb, $acc_mb, $h56k.16b, $t3.16b                    @ GHASH block 8k+2, 8k+3 - mid
6849
6850         trn1    $t6.2d, $res5.2d, $res4.2d                              @ GHASH block 8k+4, 8k+5 - mid
6851
6852         aese    $ctr0b, $rk6  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 6
6853         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
6854         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
6855         aese    $ctr6b, $rk6  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 6
6856
6857         aese    $ctr5b, $rk6  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 6
6858         aese    $ctr7b, $rk6  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 6
6859
6860         pmull2  $t4.1q, $res4.2d, $h4.2d                                @ GHASH block 8k+4 - high
6861         pmull2  $t5.1q, $res5.2d, $h3.2d                                @ GHASH block 8k+5 - high
6862         pmull   $h4.1q, $res4.1d, $h4.1d                                @ GHASH block 8k+4 - low
6863
6864         trn2    $res4.2d, $res5.2d, $res4.2d                            @ GHASH block 8k+4, 8k+5 - mid
6865         pmull   $h3.1q, $res5.1d, $h3.1d                                @ GHASH block 8k+5 - low
6866         trn1    $t9.2d, $res7.2d, $res6.2d                              @ GHASH block 8k+6, 8k+7 - mid
6867
6868         aese    $ctr7b, $rk7  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 7
6869         pmull2  $t7.1q, $res6.2d, $h2.2d                                @ GHASH block 8k+6 - high
6870         aese    $ctr1b, $rk6  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 6
6871
6872         aese    $ctr2b, $rk6  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 6
6873         aese    $ctr3b, $rk6  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 6
6874         aese    $ctr4b, $rk6  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 6
6875
6876         ldp     $rk8q, $rk9q, [$cc, #128]                               @ load rk8, rk9
6877         pmull   $h2.1q, $res6.1d, $h2.1d                                @ GHASH block 8k+6 - low
6878         aese    $ctr5b, $rk7  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 7
6879
6880         aese    $ctr1b, $rk7  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 7
6881         aese    $ctr4b, $rk7  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 7
6882
6883         aese    $ctr6b, $rk7  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 7
6884         aese    $ctr2b, $rk7  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 7
6885         eor3    $acc_hb, $acc_hb, $t4.16b, $t5.16b                      @ GHASH block 8k+4, 8k+5 - high
6886
6887         aese    $ctr0b, $rk7  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 7
6888         trn2    $res6.2d, $res7.2d, $res6.2d                            @ GHASH block 8k+6, 8k+7 - mid
6889         aese    $ctr3b, $rk7  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 7
6890
6891         aese    $ctr0b, $rk8  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 8
6892         aese    $ctr7b, $rk8  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 8
6893         aese    $ctr4b, $rk8  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 8
6894
6895         aese    $ctr1b, $rk8  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 8
6896         aese    $ctr5b, $rk8  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 8
6897         aese    $ctr6b, $rk8  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 8
6898
6899         aese    $ctr3b, $rk8  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 8
6900         aese    $ctr4b, $rk9  \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 9
6901         eor     $res4.16b, $res4.16b, $t6.16b                           @ GHASH block 8k+4, 8k+5 - mid
6902
6903         aese    $ctr0b, $rk9  \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 9
6904         aese    $ctr1b, $rk9  \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 9
6905         eor     $res6.16b, $res6.16b, $t9.16b                           @ GHASH block 8k+6, 8k+7 - mid
6906
6907         aese    $ctr6b, $rk9  \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 9
6908         aese    $ctr7b, $rk9  \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 9
6909         pmull2  $t6.1q, $res4.2d, $h34k.2d                              @ GHASH block 8k+4 - mid
6910
6911         aese    $ctr2b, $rk8  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 8
6912         pmull   $h34k.1q, $res4.1d, $h34k.1d                            @ GHASH block 8k+5 - mid
6913         pmull2  $t8.1q, $res7.2d, $h1.2d                                @ GHASH block 8k+7 - high
6914
6915         pmull2  $t9.1q, $res6.2d, $h12k.2d                              @ GHASH block 8k+6 - mid
6916         pmull   $h12k.1q, $res6.1d, $h12k.1d                            @ GHASH block 8k+7 - mid
6917         pmull   $h1.1q, $res7.1d, $h1.1d                                @ GHASH block 8k+7 - low
6918
6919         ldp     $rk10q, $rk11q, [$cc, #160]                             @ load rk10, rk11
6920         eor3    $acc_lb, $acc_lb, $h4.16b, $h3.16b                      @ GHASH block 8k+4, 8k+5 - low
6921         eor3    $acc_mb, $acc_mb, $h34k.16b, $t6.16b                    @ GHASH block 8k+4, 8k+5 - mid
6922
6923         aese    $ctr2b, $rk9  \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 9
6924         aese    $ctr3b, $rk9  \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 9
6925         aese    $ctr5b, $rk9  \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 9
6926
6927         eor3    $acc_hb, $acc_hb, $t7.16b, $t8.16b                      @ GHASH block 8k+6, 8k+7 - high
6928         eor3    $acc_lb, $acc_lb, $h2.16b, $h1.16b                      @ GHASH block 8k+6, 8k+7 - low
6929         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
6930
6931         eor3    $acc_mb, $acc_mb, $h12k.16b, $t9.16b                    @ GHASH block 8k+6, 8k+7 - mid
6932
6933         aese    $ctr4b, $rk10 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 10
6934         aese    $ctr6b, $rk10 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 10
6935         aese    $ctr5b, $rk10 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 10
6936
6937         aese    $ctr0b, $rk10 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 10
6938         aese    $ctr2b, $rk10 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 10
6939         aese    $ctr3b, $rk10 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 10
6940
6941         eor3    $acc_mb, $acc_mb, $acc_hb, $acc_lb                      @ MODULO - karatsuba tidy up
6942
6943         aese    $ctr7b, $rk10 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 10
6944         aese    $ctr1b, $rk10 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 10
6945         ldp     $rk12q, $rk13q, [$cc, #192]                             @ load rk12, rk13
6946
6947         ext     $t11.16b, $acc_hb, $acc_hb, #8                          @ MODULO - other top alignment
6948
6949         aese    $ctr2b, $rk11 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 11
6950         aese    $ctr1b, $rk11 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 11
6951         aese    $ctr0b, $rk11 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 11
6952
6953         pmull   $t12.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
6954         aese    $ctr3b, $rk11 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 11
6955
6956         aese    $ctr7b, $rk11 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 11
6957         aese    $ctr6b, $rk11 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 11
6958         aese    $ctr4b, $rk11 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 11
6959
6960         aese    $ctr5b, $rk11 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 11
6961         aese    $ctr3b, $rk12 \n  aesmc $ctr3b, $ctr3b                  @ AES block 8k+11 - round 12
6962
6963         eor3    $acc_mb, $acc_mb, $t12.16b, $t11.16b                    @ MODULO - fold into mid
6964
6965         aese    $ctr3b, $rk13                                           @ AES block 8k+11 - round 13
6966         aese    $ctr2b, $rk12 \n  aesmc $ctr2b, $ctr2b                  @ AES block 8k+10 - round 12
6967         aese    $ctr6b, $rk12 \n  aesmc $ctr6b, $ctr6b                  @ AES block 8k+14 - round 12
6968
6969         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
6970         aese    $ctr4b, $rk12 \n  aesmc $ctr4b, $ctr4b                  @ AES block 8k+12 - round 12
6971         aese    $ctr7b, $rk12 \n  aesmc $ctr7b, $ctr7b                  @ AES block 8k+15 - round 12
6972
6973         aese    $ctr0b, $rk12 \n  aesmc $ctr0b, $ctr0b                  @ AES block 8k+8 - round 12
6974         ldr     $rk14q, [$cc, #224]                                     @ load rk14
6975         aese    $ctr1b, $rk12 \n  aesmc $ctr1b, $ctr1b                  @ AES block 8k+9 - round 12
6976
6977         aese    $ctr4b, $rk13                                           @ AES block 8k+12 - round 13
6978         ext     $t11.16b, $acc_mb, $acc_mb, #8                          @ MODULO - other mid alignment
6979         aese    $ctr5b, $rk12 \n  aesmc $ctr5b, $ctr5b                  @ AES block 8k+13 - round 12
6980
6981         aese    $ctr6b, $rk13                                           @ AES block 8k+14 - round 13
6982         aese    $ctr2b, $rk13                                           @ AES block 8k+10 - round 13
6983         aese    $ctr1b, $rk13                                           @ AES block 8k+9 - round 13
6984
6985         aese    $ctr5b, $rk13                                           @ AES block 8k+13 - round 13
6986         eor3    $acc_lb, $acc_lb, $t11.16b, $acc_hb                     @ MODULO - fold into low
6987         add     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s                @ CTR block 8k+15
6988
6989         aese    $ctr7b, $rk13                                           @ AES block 8k+15 - round 13
6990         aese    $ctr0b, $rk13                                           @ AES block 8k+8 - round 13
6991 .L256_dec_tail:                                                         @ TAIL
6992
6993         ext     $t0.16b, $acc_lb, $acc_lb, #8                           @ prepare final partial tag
6994         sub     $main_end_input_ptr, $end_input_ptr, $input_ptr         @ main_end_input_ptr is number of bytes left to process
6995         cmp     $main_end_input_ptr, #112
6996
6997         ldr     $res1q, [$input_ptr], #16                               @ AES block 8k+8 - load ciphertext
6998
6999         ldp     $h78kq, $h8q, [$current_tag, #192]                      @ load h8k | h7k
7000         ext     $h8.16b, $h8.16b, $h8.16b, #8
7001         mov     $t1.16b, $rk14
7002
7003         ldp     $h5q, $h56kq, [$current_tag, #128]                      @ load h5l | h5h
7004         ext     $h5.16b, $h5.16b, $h5.16b, #8
7005
7006         eor3    $res4b, $res1b, $ctr0b, $t1.16b                         @ AES block 8k+8 - result
7007         ldp     $h6q, $h7q, [$current_tag, #160]                        @ load h6l | h6h
7008         ext     $h6.16b, $h6.16b, $h6.16b, #8
7009         ext     $h7.16b, $h7.16b, $h7.16b, #8
7010         b.gt    .L256_dec_blocks_more_than_7
7011
7012         mov     $ctr7b, $ctr6b
7013         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
7014         mov     $ctr6b, $ctr5b
7015
7016         mov     $ctr5b, $ctr4b
7017         mov     $ctr4b, $ctr3b
7018         movi    $acc_l.8b, #0
7019
7020         movi    $acc_h.8b, #0
7021         movi    $acc_m.8b, #0
7022         mov     $ctr3b, $ctr2b
7023
7024         cmp     $main_end_input_ptr, #96
7025         mov     $ctr2b, $ctr1b
7026         b.gt    .L256_dec_blocks_more_than_6
7027
7028         mov     $ctr7b, $ctr6b
7029         mov     $ctr6b, $ctr5b
7030
7031         mov     $ctr5b, $ctr4b
7032         cmp     $main_end_input_ptr, #80
7033         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
7034
7035         mov     $ctr4b, $ctr3b
7036         mov     $ctr3b, $ctr1b
7037         b.gt    .L256_dec_blocks_more_than_5
7038
7039         cmp     $main_end_input_ptr, #64
7040         mov     $ctr7b, $ctr6b
7041         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
7042
7043         mov     $ctr6b, $ctr5b
7044
7045         mov     $ctr5b, $ctr4b
7046         mov     $ctr4b, $ctr1b
7047         b.gt    .L256_dec_blocks_more_than_4
7048
7049         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
7050         mov     $ctr7b, $ctr6b
7051         cmp     $main_end_input_ptr, #48
7052
7053         mov     $ctr6b, $ctr5b
7054         mov     $ctr5b, $ctr1b
7055         b.gt    .L256_dec_blocks_more_than_3
7056
7057         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
7058         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
7059         mov     $ctr7b, $ctr6b
7060
7061         cmp     $main_end_input_ptr, #32
7062         mov     $ctr6b, $ctr1b
7063         b.gt    .L256_dec_blocks_more_than_2
7064
7065         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
7066
7067         mov     $ctr7b, $ctr1b
7068         cmp     $main_end_input_ptr, #16
7069         b.gt    .L256_dec_blocks_more_than_1
7070
7071         sub     $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s
7072         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
7073         b        .L256_dec_blocks_less_than_1
7074 .L256_dec_blocks_more_than_7:                                           @ blocks left >  7
7075         rev64   $res0b, $res1b                                          @ GHASH final-7 block
7076         ldr     $res1q, [$input_ptr], #16                               @ AES final-6 block - load ciphertext
7077         st1     { $res4b}, [$output_ptr], #16                           @ AES final-7 block  - store result
7078
7079         ins     $acc_m.d[0], $h78k.d[1]                                 @ GHASH final-7 block - mid
7080
7081         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
7082
7083         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-7 block - mid
7084         eor3    $res4b, $res1b, $ctr1b, $t1.16b                         @ AES final-6 block - result
7085
7086         pmull2  $acc_h.1q, $res0.2d, $h8.2d                             @ GHASH final-7 block - high
7087
7088         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-7 block - mid
7089         movi    $t0.8b, #0                                              @ supress further partial tag feed in
7090
7091         pmull   $acc_l.1q, $res0.1d, $h8.1d                             @ GHASH final-7 block - low
7092         pmull   $acc_m.1q, $rk4v.1d, $acc_m.1d                          @ GHASH final-7 block - mid
7093 .L256_dec_blocks_more_than_6:                                           @ blocks left >  6
7094
7095         rev64   $res0b, $res1b                                          @ GHASH final-6 block
7096
7097         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
7098         ldr     $res1q, [$input_ptr], #16                               @ AES final-5 block - load ciphertext
7099         movi    $t0.8b, #0                                              @ supress further partial tag feed in
7100
7101         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-6 block - mid
7102         st1     { $res4b}, [$output_ptr], #16                           @ AES final-6 block - store result
7103         pmull2  $rk2q1, $res0.2d, $h7.2d                                @ GHASH final-6 block - high
7104
7105         pmull   $rk3q1, $res0.1d, $h7.1d                                @ GHASH final-6 block - low
7106
7107         eor3    $res4b, $res1b, $ctr2b, $t1.16b                         @ AES final-5 block - result
7108         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-6 block - low
7109         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-6 block - mid
7110
7111         pmull   $rk4v.1q, $rk4v.1d, $h78k.1d                            @ GHASH final-6 block - mid
7112
7113         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-6 block - mid
7114         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-6 block - high
7115 .L256_dec_blocks_more_than_5:                                           @ blocks left >  5
7116
7117         rev64   $res0b, $res1b                                          @ GHASH final-5 block
7118
7119         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
7120
7121         pmull2  $rk2q1, $res0.2d, $h6.2d                                @ GHASH final-5 block - high
7122         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-5 block - mid
7123
7124         ldr     $res1q, [$input_ptr], #16                               @ AES final-4 block - load ciphertext
7125
7126         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-5 block - mid
7127         st1     { $res4b}, [$output_ptr], #16                           @ AES final-5 block - store result
7128
7129         pmull   $rk3q1, $res0.1d, $h6.1d                                @ GHASH final-5 block - low
7130         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-5 block - mid
7131
7132         pmull2  $rk4v.1q, $rk4v.2d, $h56k.2d                            @ GHASH final-5 block - mid
7133
7134         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-5 block - high
7135         eor3    $res4b, $res1b, $ctr3b, $t1.16b                         @ AES final-4 block - result
7136         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-5 block - low
7137
7138         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-5 block - mid
7139         movi    $t0.8b, #0                                              @ supress further partial tag feed in
7140 .L256_dec_blocks_more_than_4:                                           @ blocks left >  4
7141
7142         rev64   $res0b, $res1b                                          @ GHASH final-4 block
7143
7144         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
7145
7146         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-4 block - mid
7147         ldr     $res1q, [$input_ptr], #16                               @ AES final-3 block - load ciphertext
7148
7149         movi    $t0.8b, #0                                              @ supress further partial tag feed in
7150
7151         pmull   $rk3q1, $res0.1d, $h5.1d                                @ GHASH final-4 block - low
7152         pmull2  $rk2q1, $res0.2d, $h5.2d                                @ GHASH final-4 block - high
7153
7154         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-4 block - mid
7155
7156         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-4 block - high
7157
7158         pmull   $rk4v.1q, $rk4v.1d, $h56k.1d                            @ GHASH final-4 block - mid
7159
7160         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-4 block - low
7161         st1     { $res4b}, [$output_ptr], #16                           @ AES final-4 block - store result
7162
7163         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-4 block - mid
7164         eor3    $res4b, $res1b, $ctr4b, $t1.16b                         @ AES final-3 block - result
7165 .L256_dec_blocks_more_than_3:                                           @ blocks left >  3
7166
7167         ldr     $h4q, [$current_tag, #112]                              @ load h4l | h4h
7168         ext     $h4.16b, $h4.16b, $h4.16b, #8
7169         rev64   $res0b, $res1b                                          @ GHASH final-3 block
7170
7171         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
7172         ldr     $res1q, [$input_ptr], #16                               @ AES final-2 block - load ciphertext
7173         ldr     $h34kq, [$current_tag, #96]                             @ load h4k | h3k
7174
7175         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-3 block - mid
7176         st1     { $res4b}, [$output_ptr], #16                           @ AES final-3 block - store result
7177
7178         eor3    $res4b, $res1b, $ctr5b, $t1.16b                         @ AES final-2 block - result
7179
7180         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-3 block - mid
7181
7182         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-3 block - mid
7183         pmull   $rk3q1, $res0.1d, $h4.1d                                @ GHASH final-3 block - low
7184         pmull2  $rk2q1, $res0.2d, $h4.2d                                @ GHASH final-3 block - high
7185
7186         movi    $t0.8b, #0                                              @ supress further partial tag feed in
7187         pmull2  $rk4v.1q, $rk4v.2d, $h34k.2d                            @ GHASH final-3 block - mid
7188         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-3 block - low
7189
7190         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-3 block - high
7191
7192         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-3 block - mid
7193 .L256_dec_blocks_more_than_2:                                           @ blocks left >  2
7194
7195         rev64   $res0b, $res1b                                          @ GHASH final-2 block
7196
7197         ldr     $h3q, [$current_tag, #80]                               @ load h3l | h3h
7198         ext     $h3.16b, $h3.16b, $h3.16b, #8
7199         ldr     $res1q, [$input_ptr], #16                               @ AES final-1 block - load ciphertext
7200
7201         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
7202
7203         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-2 block - mid
7204
7205         pmull   $rk3q1, $res0.1d, $h3.1d                                @ GHASH final-2 block - low
7206         st1     { $res4b}, [$output_ptr], #16                           @ AES final-2 block - store result
7207         eor3    $res4b, $res1b, $ctr6b, $t1.16b                         @ AES final-1 block - result
7208
7209         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-2 block - mid
7210         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-2 block - low
7211         movi    $t0.8b, #0                                              @ supress further partial tag feed in
7212
7213         pmull   $rk4v.1q, $rk4v.1d, $h34k.1d                            @ GHASH final-2 block - mid
7214         pmull2  $rk2q1, $res0.2d, $h3.2d                                @ GHASH final-2 block - high
7215
7216         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-2 block - mid
7217         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-2 block - high
7218 .L256_dec_blocks_more_than_1:                                           @ blocks left >  1
7219
7220         rev64   $res0b, $res1b                                          @ GHASH final-1 block
7221
7222         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
7223
7224         ins     $rk4v.d[0], $res0.d[1]                                  @ GHASH final-1 block - mid
7225         ldr     $h2q, [$current_tag, #64]                               @ load h2l | h2h
7226         ext     $h2.16b, $h2.16b, $h2.16b, #8
7227
7228         eor     $rk4v.8b, $rk4v.8b, $res0.8b                            @ GHASH final-1 block - mid
7229         ldr     $res1q, [$input_ptr], #16                               @ AES final block - load ciphertext
7230         st1     { $res4b}, [$output_ptr], #16                           @ AES final-1 block - store result
7231
7232         ldr     $h12kq, [$current_tag, #48]                             @ load h2k | h1k
7233         pmull   $rk3q1, $res0.1d, $h2.1d                                @ GHASH final-1 block - low
7234
7235         ins     $rk4v.d[1], $rk4v.d[0]                                  @ GHASH final-1 block - mid
7236
7237         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final-1 block - low
7238
7239         eor3    $res4b, $res1b, $ctr7b, $t1.16b                         @ AES final block - result
7240         pmull2  $rk2q1, $res0.2d, $h2.2d                                @ GHASH final-1 block - high
7241
7242         pmull2  $rk4v.1q, $rk4v.2d, $h12k.2d                            @ GHASH final-1 block - mid
7243
7244         movi    $t0.8b, #0                                              @ supress further partial tag feed in
7245         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final-1 block - high
7246
7247         eor     $acc_mb, $acc_mb, $rk4v.16b                             @ GHASH final-1 block - mid
7248 .L256_dec_blocks_less_than_1:                                           @ blocks left <= 1
7249
7250         ld1     { $rk0}, [$output_ptr]                                  @ load existing bytes where the possibly partial last block is to be stored
7251         mvn     $temp0_x, xzr                                           @ temp0_x = 0xffffffffffffffff
7252         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
7253
7254         sub     $bit_length, $bit_length, #128                          @ bit_length -= 128
7255         rev32   $rtmp_ctr.16b, $rtmp_ctr.16b
7256         str     $rtmp_ctrq, [$counter]                                  @ store the updated counter
7257
7258         neg     $bit_length, $bit_length                                @ bit_length = 128 - #bits in input (in range [1,128])
7259
7260         and     $bit_length, $bit_length, #127                          @ bit_length %= 128
7261
7262         lsr     $temp0_x, $temp0_x, $bit_length                         @ temp0_x is mask for top 64b of last block
7263         cmp     $bit_length, #64
7264         mvn     $temp1_x, xzr                                           @ temp1_x = 0xffffffffffffffff
7265
7266         csel    $temp3_x, $temp0_x, xzr, lt
7267         csel    $temp2_x, $temp1_x, $temp0_x, lt
7268
7269         mov     $ctr0.d[0], $temp2_x                                    @ ctr0b is mask for last block
7270         mov     $ctr0.d[1], $temp3_x
7271
7272         and     $res1b, $res1b, $ctr0b                                  @ possibly partial last block has zeroes in highest bits
7273         ldr     $h1q, [$current_tag, #32]                               @ load h1l | h1h
7274         ext     $h1.16b, $h1.16b, $h1.16b, #8
7275         bif     $res4b, $rk0, $ctr0b                                    @ insert existing bytes in top end of result before storing
7276
7277         rev64   $res0b, $res1b                                          @ GHASH final block
7278
7279         eor     $res0b, $res0b, $t0.16b                                 @ feed in partial tag
7280
7281         ins     $t0.d[0], $res0.d[1]                                    @ GHASH final block - mid
7282         pmull2  $rk2q1, $res0.2d, $h1.2d                                @ GHASH final block - high
7283
7284         eor     $t0.8b, $t0.8b, $res0.8b                                @ GHASH final block - mid
7285
7286         pmull   $rk3q1, $res0.1d, $h1.1d                                @ GHASH final block - low
7287         eor     $acc_hb, $acc_hb, $rk2                                  @ GHASH final block - high
7288
7289         pmull   $t0.1q, $t0.1d, $h12k.1d                                @ GHASH final block - mid
7290
7291         eor     $acc_mb, $acc_mb, $t0.16b                               @ GHASH final block - mid
7292         ldr     $mod_constantd, [$modulo_constant]                      @ MODULO - load modulo constant
7293         eor     $acc_lb, $acc_lb, $rk3                                  @ GHASH final block - low
7294
7295         pmull   $t11.1q, $acc_h.1d, $mod_constant.1d                    @ MODULO - top 64b align with mid
7296         eor     $t10.16b, $acc_hb, $acc_lb                              @ MODULO - karatsuba tidy up
7297
7298         ext     $acc_hb, $acc_hb, $acc_hb, #8                           @ MODULO - other top alignment
7299         st1     { $res4b}, [$output_ptr]                                @ store all 16B
7300
7301         eor     $acc_mb, $acc_mb, $t10.16b                              @ MODULO - karatsuba tidy up
7302
7303         eor     $t11.16b, $acc_hb, $t11.16b                             @ MODULO - fold into mid
7304         eor     $acc_mb, $acc_mb, $t11.16b                              @ MODULO - fold into mid
7305
7306         pmull   $acc_h.1q, $acc_m.1d, $mod_constant.1d                  @ MODULO - mid 64b align with low
7307
7308         ext     $acc_mb, $acc_mb, $acc_mb, #8                           @ MODULO - other mid alignment
7309         eor     $acc_lb, $acc_lb, $acc_hb                               @ MODULO - fold into low
7310
7311         eor     $acc_lb, $acc_lb, $acc_mb                               @ MODULO - fold into low
7312         ext     $acc_lb, $acc_lb, $acc_lb, #8
7313         rev64   $acc_lb, $acc_lb
7314         st1     { $acc_l.16b }, [$current_tag]
7315         lsr     x0, $bit_length, #3                                     @ return sizes
7316
7317         ldp     d10, d11, [sp, #16]
7318         ldp     d12, d13, [sp, #32]
7319         ldp     d14, d15, [sp, #48]
7320         ldp     d8, d9, [sp], #80
7321         ret
7322
7323 .L256_dec_ret:
7324         mov w0, #0x0
7325         ret
7326 .size unroll8_eor3_aes_gcm_dec_256_kernel,.-unroll8_eor3_aes_gcm_dec_256_kernel
7327 ___
7328 }
7329 }
7330
7331 $code.=<<___;
7332 .asciz  "AES GCM module for ARMv8, SPDX BSD-3-Clause by <xiaokang.qian\@arm.com>"
7333 .align  2
7334 #endif
7335 ___
7336
7337 {
7338     my  %opcode = (
7339     "rax1"    => 0xce608c00,    "eor3"    => 0xce000000,
7340     "bcax"    => 0xce200000,    "xar"    => 0xce800000    );
7341
7342     sub unsha3 {
7343          my ($mnemonic,$arg)=@_;
7344
7345          $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv#]([0-9\-]+))?)?/
7346          &&
7347          sprintf ".inst\t0x%08x\t//%s %s",
7348             $opcode{$mnemonic}|$1|($2<<5)|($3<<16)|(eval($4)<<10),
7349             $mnemonic,$arg;
7350     }
7351     sub unvmov {
7352         my $arg=shift;
7353
7354         $arg =~ m/q([0-9]+)#(lo|hi),\s*q([0-9]+)#(lo|hi)/o &&
7355         sprintf "ins    v%d.d[%d],v%d.d[%d]",$1<8?$1:$1+8,($2 eq "lo")?0:1,
7356                              $3<8?$3:$3+8,($4 eq "lo")?0:1;
7357     }
7358
7359      foreach(split("\n",$code)) {
7360         s/@\s/\/\//o;               # old->new style commentary
7361         s/\`([^\`]*)\`/eval($1)/ge;
7362
7363         m/\bld1r\b/ and s/\.16b/.2d/g    or
7364         s/\b(eor3|rax1|xar|bcax)\s+(v.*)/unsha3($1,$2)/ge;
7365         print $_,"\n";
7366      }
7367 }
7368
7369 close STDOUT or die "error closing STDOUT: $!"; # enforce flush