--- /dev/null
+#! /usr/bin/env perl
+# This file is dual-licensed, meaning that you can use it under your
+# choice of either of the following two licenses:
+#
+# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License"). You can obtain
+# a copy in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+#
+# or
+#
+# Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# - RV64I
+# - RISC-V Vector ('V') with VLEN >= 128
+# - RISC-V Vector Basic Bit-manipulation extension ('Zvbb')
+# - RISC-V Vector GCM/GMAC extension ('Zvkg')
+# - RISC-V Vector AES Block Cipher extension ('Zvkned')
+# - RISC-V Zicclsm(Main memory supports misaligned loads/stores)
+
+# Reference: https://github.com/riscv/riscv-crypto/issues/192#issuecomment-1270447575
+#
+# Assume we have 12 GCM blocks and we try to parallelize GCM computation for 4 blocks.
+# Tag = M0*H^12 + M1*H^11 + M2*H^10 + M3*H^9 +
+# M4*H^8 + M5*H^7 + M6*H^6 + M7*H^5 +
+# M8*H^4 + M9*H^3 + M10*H^2 + M11*H^1
+# We could rewrite the formula into:
+# T0 = 0
+# T1 = (T0+M1)*H^4 T2 = (T0+M2)*H^4 T3 = (T0+M3)*H^4 T4 = (T0+M4)*H^4
+# T5 = (T1+M5)*H^4 T6 = (T2+M6)*H^4 T7 = (T3+M7)*H^4 T8 = (T4+M8)*H^4
+# T9 = (T5+M9)*H^4 T10 = (T6+M10)*H^3 T11 = (T7+M11)*H^2 T12 = (T8+M12)*H^1
+#
+# We will multiply with [H^4, H^4, H^4, H^4] in each steps except the last iteration.
+# The last iteration will multiply with [H^4, H^3, H^2, H^1].
+
+use strict;
+use warnings;
+
+use FindBin qw($Bin);
+use lib "$Bin";
+use lib "$Bin/../../perlasm";
+use riscv;
+
+# $output is the last argument if it looks like a file (it has an extension)
+# $flavour is the first argument if it doesn't look like a file
+my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
+my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
+
+$output and open STDOUT,">$output";
+
+my $code=<<___;
+.text
+___
+
+{
+my ($INP, $OUTP, $LEN, $KEYP, $IVP, $XIP) = ("a0", "a1", "a2", "a3", "a4", "a5");
+my ($T0, $T1, $T2, $T3) = ("t0", "t1", "t2", "t3");
+my ($PADDING_LEN32) = ("t4");
+my ($LEN32) = ("t5");
+my ($CTR) = ("t6");
+my ($FULL_BLOCK_LEN32) = ("a6");
+my ($ORIGINAL_LEN32) = ("a7");
+my ($PROCESSED_LEN) = ("a0");
+my ($CTR_MASK) = ("v0");
+my ($INPUT_PADDING_MASK) = ("v0");
+my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7,
+ $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15,
+ $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23,
+ $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31,
+) = map("v$_",(0..31));
+
+# Do aes-128 enc.
+sub aes_128_cipher_body {
+ my $code=<<___;
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vaesz_vs $V28, $V1]}
+ @{[vaesem_vs $V28, $V2]}
+ @{[vaesem_vs $V28, $V3]}
+ @{[vaesem_vs $V28, $V4]}
+ @{[vaesem_vs $V28, $V5]}
+ @{[vaesem_vs $V28, $V6]}
+ @{[vaesem_vs $V28, $V7]}
+ @{[vaesem_vs $V28, $V8]}
+ @{[vaesem_vs $V28, $V9]}
+ @{[vaesem_vs $V28, $V10]}
+ @{[vaesef_vs $V28, $V11]}
+___
+
+ return $code;
+}
+
+# Do aes-192 enc.
+sub aes_192_cipher_body {
+ my $TMP_REG = shift;
+
+ my $code=<<___;
+ # Load key 4
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ addi $TMP_REG, $KEYP, 48
+ @{[vle32_v $V11, $TMP_REG]}
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vaesz_vs $V28, $V1]}
+ @{[vaesem_vs $V28, $V2]}
+ @{[vaesem_vs $V28, $V3]}
+ @{[vaesem_vs $V28, $V11]}
+ # Load key 8
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ addi $TMP_REG, $KEYP, 112
+ @{[vle32_v $V11, $TMP_REG]}
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vaesem_vs $V28, $V4]}
+ @{[vaesem_vs $V28, $V5]}
+ @{[vaesem_vs $V28, $V6]}
+ @{[vaesem_vs $V28, $V11]}
+ # Load key 13
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ addi $TMP_REG, $KEYP, 192
+ @{[vle32_v $V11, $TMP_REG]}
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vaesem_vs $V28, $V7]}
+ @{[vaesem_vs $V28, $V8]}
+ @{[vaesem_vs $V28, $V9]}
+ @{[vaesem_vs $V28, $V10]}
+ @{[vaesef_vs $V28, $V11]}
+___
+
+ return $code;
+}
+
+# Do aes-256 enc.
+sub aes_256_cipher_body {
+ my $TMP_REG = shift;
+
+ my $code=<<___;
+ # Load key 3
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ addi $TMP_REG, $KEYP, 32
+ @{[vle32_v $V11, $TMP_REG]}
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vaesz_vs $V28, $V1]}
+ @{[vaesem_vs $V28, $V2]}
+ @{[vaesem_vs $V28, $V11]}
+ # Load key 6
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ addi $TMP_REG, $KEYP, 80
+ @{[vle32_v $V11, $TMP_REG]}
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vaesem_vs $V28, $V3]}
+ @{[vaesem_vs $V28, $V4]}
+ @{[vaesem_vs $V28, $V11]}
+ # Load key 9
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ addi $TMP_REG, $KEYP, 128
+ @{[vle32_v $V11, $TMP_REG]}
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vaesem_vs $V28, $V5]}
+ @{[vaesem_vs $V28, $V6]}
+ @{[vaesem_vs $V28, $V11]}
+ # Load key 12
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ addi $TMP_REG, $KEYP, 176
+ @{[vle32_v $V11, $TMP_REG]}
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vaesem_vs $V28, $V7]}
+ @{[vaesem_vs $V28, $V8]}
+ @{[vaesem_vs $V28, $V11]}
+ # Load key 15
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ addi $TMP_REG, $KEYP, 224
+ @{[vle32_v $V11, $TMP_REG]}
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vaesem_vs $V28, $V9]}
+ @{[vaesem_vs $V28, $V10]}
+ @{[vaesef_vs $V28, $V11]}
+___
+
+ return $code;
+}
+
+sub handle_padding_in_first_round {
+ my $TMP_REG = shift;
+
+ my $code=<<___;
+ bnez $PADDING_LEN32, 1f
+
+ ## without padding
+ # Store ciphertext/plaintext
+ @{[vse32_v $V28, $OUTP]}
+ j 2f
+
+ ## with padding
+1:
+ # Store ciphertext/plaintext using mask
+ @{[vse32_v $V28, $OUTP, $INPUT_PADDING_MASK]}
+
+ # Fill zero for the padding blocks
+ @{[vsetvli "zero", $PADDING_LEN32, "e32", "m4", "tu", "ma"]}
+ @{[vmv_v_i $V28, 0]}
+
+ # We have used mask register for `INPUT_PADDING_MASK` before. We need to
+ # setup the ctr mask back.
+ # ctr mask : [000100010001....]
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e8", "m1", "ta", "ma"]}
+ li $TMP_REG, 0b10001000
+ @{[vmv_v_x $CTR_MASK, $TMP_REG]}
+2:
+
+___
+
+ return $code;
+}
+
+
+# Do aes-128 enc for first round.
+sub aes_128_first_round {
+ my $PTR_OFFSET_REG = shift;
+ my $TMP_REG = shift;
+
+ my $code=<<___;
+ # Load all 11 aes round keys to v1-v11 registers.
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ @{[vle32_v $V1, $KEYP]}
+ addi $KEYP, $KEYP, 16
+ @{[vle32_v $V2, $KEYP]}
+ addi $KEYP, $KEYP, 16
+ @{[vle32_v $V3, $KEYP]}
+ addi $KEYP, $KEYP, 16
+ @{[vle32_v $V4, $KEYP]}
+ addi $KEYP, $KEYP, 16
+ @{[vle32_v $V5, $KEYP]}
+ addi $KEYP, $KEYP, 16
+ @{[vle32_v $V6, $KEYP]}
+ addi $KEYP, $KEYP, 16
+ @{[vle32_v $V7, $KEYP]}
+ addi $KEYP, $KEYP, 16
+ @{[vle32_v $V8, $KEYP]}
+ addi $KEYP, $KEYP, 16
+ @{[vle32_v $V9, $KEYP]}
+ addi $KEYP, $KEYP, 16
+ @{[vle32_v $V10, $KEYP]}
+ addi $KEYP, $KEYP, 16
+ @{[vle32_v $V11, $KEYP]}
+
+ # We already have the ciphertext/plaintext and ctr data for the first round.
+ @{[aes_128_cipher_body]}
+
+ # Compute AES ctr result.
+ @{[vxor_vv $V28, $V28, $V24]}
+
+ @{[handle_padding_in_first_round $TMP_REG]}
+
+ add $INP, $INP, $PTR_OFFSET_REG
+ add $OUTP, $OUTP, $PTR_OFFSET_REG
+___
+
+ return $code;
+}
+
+# Do aes-192 enc for first round.
+sub aes_192_first_round {
+ my $PTR_OFFSET_REG = shift;
+ my $TMP_REG = shift;
+
+ my $code=<<___;
+ # We run out of 32 vector registers, so we just preserve some round keys
+ # and load the remaining round keys inside the aes body.
+ # We keep the round keys for:
+ # 1, 2, 3, 5, 6, 7, 9, 10, 11 and 12th keys.
+ # The following keys will be loaded in the aes body:
+ # 4, 8 and 13th keys.
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ # key 1
+ @{[vle32_v $V1, $KEYP]}
+ # key 2
+ addi $TMP_REG, $KEYP, 16
+ @{[vle32_v $V2, $TMP_REG]}
+ # key 3
+ addi $TMP_REG, $KEYP, 32
+ @{[vle32_v $V3, $TMP_REG]}
+ # key 5
+ addi $TMP_REG, $KEYP, 64
+ @{[vle32_v $V4, $TMP_REG]}
+ # key 6
+ addi $TMP_REG, $KEYP, 80
+ @{[vle32_v $V5, $TMP_REG]}
+ # key 7
+ addi $TMP_REG, $KEYP, 96
+ @{[vle32_v $V6, $TMP_REG]}
+ # key 9
+ addi $TMP_REG, $KEYP, 128
+ @{[vle32_v $V7, $TMP_REG]}
+ # key 10
+ addi $TMP_REG, $KEYP, 144
+ @{[vle32_v $V8, $TMP_REG]}
+ # key 11
+ addi $TMP_REG, $KEYP, 160
+ @{[vle32_v $V9, $TMP_REG]}
+ # key 12
+ addi $TMP_REG, $KEYP, 176
+ @{[vle32_v $V10, $TMP_REG]}
+
+ # We already have the ciphertext/plaintext and ctr data for the first round.
+ @{[aes_192_cipher_body $TMP_REG]}
+
+ # Compute AES ctr result.
+ @{[vxor_vv $V28, $V28, $V24]}
+
+ @{[handle_padding_in_first_round $TMP_REG]}
+
+ add $INP, $INP, $PTR_OFFSET_REG
+ add $OUTP, $OUTP, $PTR_OFFSET_REG
+___
+
+ return $code;
+}
+
+# Do aes-256 enc for first round.
+sub aes_256_first_round {
+ my $PTR_OFFSET_REG = shift;
+ my $TMP_REG = shift;
+
+ my $code=<<___;
+ # We run out of 32 vector registers, so we just preserve some round keys
+ # and load the remaining round keys inside the aes body.
+ # We keep the round keys for:
+ # 1, 2, 4, 5, 7, 8, 10, 11, 13 and 14th keys.
+ # The following keys will be loaded in the aes body:
+ # 3, 6, 9, 12 and 15th keys.
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ # key 1
+ @{[vle32_v $V1, $KEYP]}
+ # key 2
+ addi $TMP_REG, $KEYP, 16
+ @{[vle32_v $V2, $TMP_REG]}
+ # key 4
+ addi $TMP_REG, $KEYP, 48
+ @{[vle32_v $V3, $TMP_REG]}
+ # key 5
+ addi $TMP_REG, $KEYP, 64
+ @{[vle32_v $V4, $TMP_REG]}
+ # key 7
+ addi $TMP_REG, $KEYP, 96
+ @{[vle32_v $V5, $TMP_REG]}
+ # key 8
+ addi $TMP_REG, $KEYP, 112
+ @{[vle32_v $V6, $TMP_REG]}
+ # key 10
+ addi $TMP_REG, $KEYP, 144
+ @{[vle32_v $V7, $TMP_REG]}
+ # key 11
+ addi $TMP_REG, $KEYP, 160
+ @{[vle32_v $V8, $TMP_REG]}
+ # key 13
+ addi $TMP_REG, $KEYP, 192
+ @{[vle32_v $V9, $TMP_REG]}
+ # key 14
+ addi $TMP_REG, $KEYP, 208
+ @{[vle32_v $V10, $TMP_REG]}
+
+ # We already have the ciphertext/plaintext and ctr data for the first round.
+ @{[aes_256_cipher_body $TMP_REG]}
+
+ # Compute AES ctr result.
+ @{[vxor_vv $V28, $V28, $V24]}
+
+ @{[handle_padding_in_first_round $TMP_REG]}
+
+ add $INP, $INP, $PTR_OFFSET_REG
+ add $OUTP, $OUTP, $PTR_OFFSET_REG
+___
+
+ return $code;
+}
+
+sub aes_gcm_init {
+ my $code=<<___;
+ # Compute the AES-GCM full-block e32 length for `LMUL=4`. We will handle
+ # the multiple AES-GCM blocks at the same time within `LMUL=4` register.
+ # The AES-GCM's SEW is e32 and EGW is 128 bits.
+ # FULL_BLOCK_LEN32 = (VLEN*LMUL)/(EGW) * (EGW/SEW) = (VLEN*4)/(32*4) * 4
+ # = (VLEN*4)/32
+ # We could get the block_num using the VL value of `vsetvli with e32, m4`.
+ @{[vsetvli $FULL_BLOCK_LEN32, "zero", "e32", "m4", "ta", "ma"]}
+ # If `LEN32 % FULL_BLOCK_LEN32` is not equal to zero, we could fill the
+ # zero padding data to make sure we could always handle FULL_BLOCK_LEN32
+ # blocks for all iterations.
+
+ ## Prepare the H^n multiplier in v16 for GCM multiplier. The `n` is the gcm
+ ## block number in a LMUL=4 register group.
+ ## n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4))
+ ## = (VLEN/32)
+ ## We could use vsetvli with `e32, m1` to compute the `n` number.
+ @{[vsetvli $T0, "zero", "e32", "m1", "ta", "ma"]}
+
+ # The H is at `gcm128_context.Htable[0]`(addr(Xi)+16*2).
+ addi $T1, $XIP, 32
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ @{[vle32_v $V31, $T1]}
+
+ # Compute the H^n
+ li $T1, 1
+1:
+ @{[vgmul_vv $V31, $V31]}
+ slli $T1, $T1, 1
+ bltu $T1, $T0, 1b
+
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vmv_v_i $V16, 0]}
+ @{[vaesz_vs $V16, $V31]}
+
+ #### Load plaintext into v24 and handle padding. We also load the init tag
+ #### data into v20 and prepare the AES ctr input data into v12 and v28.
+ @{[vmv_v_i $V20, 0]}
+
+ ## Prepare the AES ctr input data into v12.
+ # Setup ctr input mask.
+ # ctr mask : [000100010001....]
+ # Note: The actual vl should be `FULL_BLOCK_LEN32/4 * 2`, but we just use
+ # `FULL_BLOCK_LEN32` here.
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e8", "m1", "ta", "ma"]}
+ li $T0, 0b10001000
+ @{[vmv_v_x $CTR_MASK, $T0]}
+ # Load IV.
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ @{[vle32_v $V31, $IVP]}
+ # Convert the big-endian counter into little-endian.
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "mu"]}
+ @{[vrev8_v $V31, $V31, $CTR_MASK]}
+ # Splat the `single block of IV` to v12
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vmv_v_i $V12, 0]}
+ @{[vaesz_vs $V12, $V31]}
+ # Prepare the ctr counter into v8
+ # v8: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...]
+ @{[viota_m $V8, $CTR_MASK, $CTR_MASK]}
+ # Merge IV and ctr counter into v12.
+ # v12:[x, x, x, count+0, x, x, x, count+1, ...]
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
+ @{[vadd_vv $V12, $V12, $V8, $CTR_MASK]}
+
+ li $PADDING_LEN32, 0
+ # Get the SEW32 size in the first round.
+ # If we have the non-zero value for `LEN32&(FULL_BLOCK_LEN32-1)`, then
+ # we will have the leading padding zero.
+ addi $T0, $FULL_BLOCK_LEN32, -1
+ and $T0, $T0, $LEN32
+ beqz $T0, 1f
+
+ ## with padding
+ sub $LEN32, $LEN32, $T0
+ sub $PADDING_LEN32, $FULL_BLOCK_LEN32, $T0
+ # padding block size
+ srli $T1, $PADDING_LEN32, 2
+ # padding byte size
+ slli $T2, $PADDING_LEN32, 2
+
+ # Adjust the ctr counter to make the counter start from `counter+0` for the
+ # first non-padding block.
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
+ @{[vsub_vx $V12, $V12, $T1, $CTR_MASK]}
+ # Prepare the AES ctr input into v28.
+ # The ctr data uses big-endian form.
+ @{[vmv_v_v $V28, $V12]}
+ @{[vrev8_v $V28, $V28, $CTR_MASK]}
+
+ # Prepare the mask for input loading in the first round. We use
+ # `VL=FULL_BLOCK_LEN32` with the mask in the first round.
+ # Adjust input ptr.
+ sub $INP, $INP, $T2
+ # Adjust output ptr.
+ sub $OUTP, $OUTP, $T2
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e16", "m2", "ta", "ma"]}
+ @{[vid_v $V2]}
+ # We don't use the pseudo instruction `vmsgeu` here. Use `vmsgtu` instead.
+ # The original code is:
+ # vmsgeu.vx $INPUT_PADDING_MASK, $V2, $PADDING_LEN32
+ addi $T0, $PADDING_LEN32, -1
+ @{[vmsgtu_vx $INPUT_PADDING_MASK, $V2, $T0]}
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vmv_v_i $V24, 0]}
+ # Load the input for length FULL_BLOCK_LEN32 with mask.
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
+ @{[vle32_v $V24, $INP, $INPUT_PADDING_MASK]}
+
+ # Load the init `Xi` data to v20 with preceding zero padding.
+ # Adjust Xi ptr.
+ sub $T0, $XIP, $T2
+ # Load for length `zero-padding-e32-length + 4`.
+ addi $T1, $PADDING_LEN32, 4
+ @{[vsetvli "zero", $T1, "e32", "m4", "tu", "mu"]}
+ @{[vle32_v $V20, $T0, $INPUT_PADDING_MASK]}
+ j 2f
+
+1:
+ ## without padding
+ sub $LEN32, $LEN32, $FULL_BLOCK_LEN32
+
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+ @{[vle32_v $V24, $INP]}
+
+ # Load the init Xi data to v20.
+ @{[vsetivli "zero", 4, "e32", "m1", "tu", "ma"]}
+ @{[vle32_v $V20, $XIP]}
+
+ # Prepare the AES ctr input into v28.
+ # The ctr data uses big-endian form.
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
+ @{[vmv_v_v $V28, $V12]}
+ @{[vrev8_v $V28, $V28, $CTR_MASK]}
+2:
+___
+
+ return $code;
+}
+
+sub prepare_input_and_ctr {
+ my $PTR_OFFSET_REG = shift;
+
+ my $code=<<___;
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]}
+ # Increase ctr in v12.
+ @{[vadd_vx $V12, $V12, $CTR, $CTR_MASK]}
+ sub $LEN32, $LEN32, $FULL_BLOCK_LEN32
+ # Load plaintext into v24
+ @{[vsetvli "zero", "zero", "e32", "m4", "ta", "ma"]}
+ @{[vle32_v $V24, $INP]}
+ # Prepare the AES ctr input into v28.
+ # The ctr data uses big-endian form.
+ @{[vmv_v_v $V28, $V12]}
+ add $INP, $INP, $PTR_OFFSET_REG
+ @{[vsetvli "zero", "zero", "e32", "m4", "ta", "mu"]}
+ @{[vrev8_v $V28, $V28, $CTR_MASK]}
+___
+
+ return $code;
+}
+
+# Store the current CTR back to IV buffer.
+sub store_current_ctr {
+ my $code=<<___;
+ @{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]}
+ # Update current ctr value to v12
+ @{[vadd_vx $V12, $V12, $CTR, $CTR_MASK]}
+ # Convert ctr to big-endian counter.
+ @{[vrev8_v $V12, $V12, $CTR_MASK]}
+ @{[vse32_v $V12, $IVP, $CTR_MASK]}
+___
+
+ return $code;
+}
+
+# Compute the final tag into v0 from the partial tag v20.
+sub compute_final_tag {
+ my $TMP_REG = shift;
+
+ my $code=<<___;
+ # The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2).
+ # Load H to v1
+ addi $TMP_REG, $XIP, 32
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ @{[vle32_v $V1, $TMP_REG]}
+ # Multiply H for each partial tag and XOR them together.
+ # Handle 1st partial tag
+ @{[vmv_v_v $V0, $V20]}
+ @{[vgmul_vv $V0, $V1]}
+ # Handle 2nd to N-th partial tags
+ li $TMP_REG, 4
+1:
+ @{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]}
+ @{[vslidedown_vx $V4, $V20, $TMP_REG]}
+ @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]}
+ @{[vghsh_vv $V0, $V1, $V4]}
+ addi $TMP_REG, $TMP_REG, 4
+ blt $TMP_REG, $FULL_BLOCK_LEN32, 1b
+___
+
+ return $code;
+}
+
+################################################################################
+# size_t rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt(const unsigned char *in,
+# unsigned char *out, size_t len,
+# const void *key,
+# unsigned char ivec[16], u64 *Xi);
+{
+$code .= <<___;
+.p2align 3
+.globl rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt
+.type rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt,\@function
+rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt:
+ srli $T0, $LEN, 4
+ beqz $T0, .Lenc_end
+ slli $LEN32, $T0, 2
+
+ mv $ORIGINAL_LEN32, $LEN32
+
+ @{[aes_gcm_init]}
+
+ # Load number of rounds
+ lwu $T0, 240($KEYP)
+ li $T1, 14
+ li $T2, 12
+ li $T3, 10
+
+ beq $T0, $T1, aes_gcm_enc_blocks_256
+ beq $T0, $T2, aes_gcm_enc_blocks_192
+ beq $T0, $T3, aes_gcm_enc_blocks_128
+
+.Lenc_end:
+ li $PROCESSED_LEN, 0
+ ret
+
+.size rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt,.-rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt
+___
+
+$code .= <<___;
+.p2align 3
+aes_gcm_enc_blocks_128:
+ srli $CTR, $FULL_BLOCK_LEN32, 2
+ slli $T0, $FULL_BLOCK_LEN32, 2
+
+ @{[aes_128_first_round $T0, $T1]}
+
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+
+.Lenc_blocks_128:
+ # Compute the partial tags.
+ # The partial tags will multiply with [H^n, H^n, ..., H^n]
+ # [tag0, tag1, ...] =
+ # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
+ # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
+ beqz $LEN32, .Lenc_blocks_128_end
+ @{[vghsh_vv $V20, $V16, $V28]}
+
+ @{[prepare_input_and_ctr $T0]}
+
+ @{[aes_128_cipher_body]}
+
+ # Compute AES ctr ciphertext result.
+ @{[vxor_vv $V28, $V28, $V24]}
+
+ # Store ciphertext
+ @{[vse32_v $V28, $OUTP]}
+ add $OUTP, $OUTP, $T0
+
+ j .Lenc_blocks_128
+.Lenc_blocks_128_end:
+
+ # Add ciphertext into partial tag
+ @{[vxor_vv $V20, $V20, $V28]}
+
+ @{[store_current_ctr]}
+
+ @{[compute_final_tag $T1]}
+
+ # Save the final tag
+ @{[vse32_v $V0, $XIP]}
+
+ # return the processed size.
+ slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
+ ret
+.size aes_gcm_enc_blocks_128,.-aes_gcm_enc_blocks_128
+___
+
+$code .= <<___;
+.p2align 3
+aes_gcm_enc_blocks_192:
+ srli $CTR, $FULL_BLOCK_LEN32, 2
+ slli $T0, $FULL_BLOCK_LEN32, 2
+
+ @{[aes_192_first_round $T0, $T1]}
+
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+
+.Lenc_blocks_192:
+ # Compute the partial tags.
+ # The partial tags will multiply with [H^n, H^n, ..., H^n]
+ # [tag0, tag1, ...] =
+ # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
+ # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
+ beqz $LEN32, .Lenc_blocks_192_end
+ @{[vghsh_vv $V20, $V16, $V28]}
+
+ @{[prepare_input_and_ctr $T0]}
+
+ @{[aes_192_cipher_body $T1]}
+
+ # Compute AES ctr ciphertext result.
+ @{[vxor_vv $V28, $V28, $V24]}
+
+ # Store ciphertext
+ @{[vse32_v $V28, $OUTP]}
+ add $OUTP, $OUTP, $T0
+
+ j .Lenc_blocks_192
+.Lenc_blocks_192_end:
+
+ # Add ciphertext into partial tag
+ @{[vxor_vv $V20, $V20, $V28]}
+
+ @{[store_current_ctr]}
+
+ @{[compute_final_tag $T1]}
+
+ # Save the final tag
+ @{[vse32_v $V0, $XIP]}
+
+ # return the processed size.
+ slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
+ ret
+.size aes_gcm_enc_blocks_192,.-aes_gcm_enc_blocks_192
+___
+
+$code .= <<___;
+.p2align 3
+aes_gcm_enc_blocks_256:
+ srli $CTR, $FULL_BLOCK_LEN32, 2
+ slli $T0, $FULL_BLOCK_LEN32, 2
+
+ @{[aes_256_first_round $T0, $T1]}
+
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+
+.Lenc_blocks_256:
+ # Compute the partial tags.
+ # The partial tags will multiply with [H^n, H^n, ..., H^n]
+ # [tag0, tag1, ...] =
+ # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
+ # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
+ beqz $LEN32, .Lenc_blocks_256_end
+ @{[vghsh_vv $V20, $V16, $V28]}
+
+ @{[prepare_input_and_ctr $T0]}
+
+ @{[aes_256_cipher_body $T1]}
+
+ # Compute AES ctr ciphertext result.
+ @{[vxor_vv $V28, $V28, $V24]}
+
+ # Store ciphertext
+ @{[vse32_v $V28, $OUTP]}
+ add $OUTP, $OUTP, $T0
+
+ j .Lenc_blocks_256
+.Lenc_blocks_256_end:
+
+ # Add ciphertext into partial tag
+ @{[vxor_vv $V20, $V20, $V28]}
+
+ @{[store_current_ctr]}
+
+ @{[compute_final_tag $T1]}
+
+ # Save the final tag
+ @{[vse32_v $V0, $XIP]}
+
+ # return the processed size.
+ slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
+ ret
+.size aes_gcm_enc_blocks_256,.-aes_gcm_enc_blocks_256
+___
+
+}
+
+################################################################################
+# size_t rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt(const unsigned char *in,
+# unsigned char *out, size_t len,
+# const void *key,
+# unsigned char ivec[16], u64 *Xi);
+{
+$code .= <<___;
+.p2align 3
+.globl rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt
+.type rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt,\@function
+rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt:
+ srli $T0, $LEN, 4
+ beqz $T0, .Ldec_end
+ slli $LEN32, $T0, 2
+
+ mv $ORIGINAL_LEN32, $LEN32
+
+ @{[aes_gcm_init]}
+
+ # Load number of rounds
+ lwu $T0, 240($KEYP)
+ li $T1, 14
+ li $T2, 12
+ li $T3, 10
+
+ beq $T0, $T1, aes_gcm_dec_blocks_256
+ beq $T0, $T2, aes_gcm_dec_blocks_192
+ beq $T0, $T3, aes_gcm_dec_blocks_128
+
+.Ldec_end:
+ li $PROCESSED_LEN, 0
+ ret
+.size rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt,.-rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt
+___
+
+$code .= <<___;
+.p2align 3
+aes_gcm_dec_blocks_128:
+ srli $CTR, $FULL_BLOCK_LEN32, 2
+ slli $T0, $FULL_BLOCK_LEN32, 2
+
+ @{[aes_128_first_round $T0, $T1]}
+
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+
+.Ldec_blocks_128:
+ # Compute the partial tags.
+ # The partial tags will multiply with [H^n, H^n, ..., H^n]
+ # [tag0, tag1, ...] =
+ # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
+ # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
+ beqz $LEN32, .Ldec_blocks_256_end
+ @{[vghsh_vv $V20, $V16, $V24]}
+
+ @{[prepare_input_and_ctr $T0]}
+
+ @{[aes_128_cipher_body]}
+
+ # Compute AES ctr plaintext result.
+ @{[vxor_vv $V28, $V28, $V24]}
+
+ # Store plaintext
+ @{[vse32_v $V28, $OUTP]}
+ add $OUTP, $OUTP, $T0
+
+ j .Ldec_blocks_128
+.Ldec_blocks_128_end:
+
+ # Add ciphertext into partial tag
+ @{[vxor_vv $V20, $V20, $V24]}
+
+ @{[store_current_ctr]}
+
+ @{[compute_final_tag $T1]}
+
+ # Save the final tag
+ @{[vse32_v $V0, $XIP]}
+
+ # return the processed size.
+ slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
+ ret
+.size aes_gcm_dec_blocks_128,.-aes_gcm_dec_blocks_128
+___
+
+$code .= <<___;
+.p2align 3
+aes_gcm_dec_blocks_192:
+ srli $CTR, $FULL_BLOCK_LEN32, 2
+ slli $T0, $FULL_BLOCK_LEN32, 2
+
+ @{[aes_192_first_round $T0, $T1]}
+
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+
+.Ldec_blocks_192:
+ # Compute the partial tags.
+ # The partial tags will multiply with [H^n, H^n, ..., H^n]
+ # [tag0, tag1, ...] =
+ # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
+ # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
+ beqz $LEN32, .Ldec_blocks_192_end
+ @{[vghsh_vv $V20, $V16, $V24]}
+
+ @{[prepare_input_and_ctr $T0]}
+
+ @{[aes_192_cipher_body $T1]}
+
+ # Compute AES ctr plaintext result.
+ @{[vxor_vv $V28, $V28, $V24]}
+
+ # Store plaintext
+ @{[vse32_v $V28, $OUTP]}
+ add $OUTP, $OUTP, $T0
+
+ j .Ldec_blocks_192
+.Ldec_blocks_192_end:
+
+ # Add ciphertext into partial tag
+ @{[vxor_vv $V20, $V20, $V24]}
+
+ @{[store_current_ctr]}
+
+ @{[compute_final_tag $T1]}
+
+ # Save the final tag
+ @{[vse32_v $V0, $XIP]}
+
+ # return the processed size.
+ slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
+ ret
+.size aes_gcm_dec_blocks_192,.-aes_gcm_dec_blocks_192
+___
+
+$code .= <<___;
+.p2align 3
+aes_gcm_dec_blocks_256:
+ srli $CTR, $FULL_BLOCK_LEN32, 2
+ slli $T0, $FULL_BLOCK_LEN32, 2
+
+ @{[aes_256_first_round $T0, $T1]}
+
+ @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]}
+
+.Ldec_blocks_256:
+ # Compute the partial tags.
+ # The partial tags will multiply with [H^n, H^n, ..., H^n]
+ # [tag0, tag1, ...] =
+ # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n]
+ # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round.
+ beqz $LEN32, .Ldec_blocks_256_end
+ @{[vghsh_vv $V20, $V16, $V24]}
+
+ @{[prepare_input_and_ctr $T0]}
+
+ @{[aes_256_cipher_body $T1]}
+
+ # Compute AES ctr plaintext result.
+ @{[vxor_vv $V28, $V28, $V24]}
+
+ # Store plaintext
+ @{[vse32_v $V28, $OUTP]}
+ add $OUTP, $OUTP, $T0
+
+ j .Ldec_blocks_256
+.Ldec_blocks_256_end:
+
+ # Add ciphertext into partial tag
+ @{[vxor_vv $V20, $V20, $V24]}
+
+ @{[store_current_ctr]}
+
+ @{[compute_final_tag $T1]}
+
+ # Save the final tag
+ @{[vse32_v $V0, $XIP]}
+
+ # return the processed size.
+ slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2
+ ret
+.size aes_gcm_dec_blocks_256,.-aes_gcm_dec_blocks_256
+___
+
+}
+}
+
+print $code;
+
+close STDOUT or die "error closing STDOUT: $!";
projects / openssl.git / commitdiff