Fix GHASH-ASM implementation on s390x

s390x GHASH assembler implementation assumed it was called from a
gcm128_context structure where the Xi paramter to the ghash function was
embedded in that structure.  Since the structure layout resembles the paramter
block required for kimd-GHASH, the assembler code simply assumed the 128 bytes
after Xi are the hash subkey.

This assumption was broken with the introduction of AES-GCM-SIV which uses the
GHASH implementation without a gcm128_context structure.  Furthermore, the
bytes following the Xi input parameter to the GHASH function do not contain
the hash subkey.  To fix this, we remove the assumption about the calling
context and build the parameter block on the stack.  This requires some
copying of data to and from the stack.  While this introduces a performance
degradation, new systems anyway use kma for GHASH/AES-GCM.

Finally fixes #18693 for s390x.

Signed-off-by: Juergen Christ <jchrist@linux.ibm.com>
Reviewed-by: Todd Short <todd.short@me.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/18939)

diff --git a/crypto/modes/asm/ghash-s390x.pl b/crypto/modes/asm/ghash-s390x.pl
index ba9c5b4a4f4fb5890cca2e39638e659478361472..0ae6adbf6146faacfa2b5dee0a8c8433aaf80e8d 100644 (file)
--- a/crypto/modes/asm/ghash-s390x.pl
+++ b/crypto/modes/asm/ghash-s390x.pl
@@ -90,25 +90,6 @@ $code.=<<___;
 .align 32
 gcm_gmult_4bit:
 ___
-$code.=<<___ if(!$softonly && 0);      # hardware is slow for single block...
-       larl    %r1,OPENSSL_s390xcap_P
-       lghi    %r0,0
-       lg      %r1,S390X_KIMD+8(%r1)   # load second word of kimd capabilities
-                                       #  vector
-       tmhh    %r1,0x4000      # check for function 65
-       jz      .Lsoft_gmult
-       stg     %r0,16($sp)     # arrange 16 bytes of zero input
-       stg     %r0,24($sp)
-       lghi    %r0,S390X_GHASH # function 65
-       la      %r1,0($Xi)      # H lies right after Xi in gcm128_context
-       la      $inp,16($sp)
-       lghi    $len,16
-       .long   0xb93e0004      # kimd %r0,$inp
-       brc     1,.-4           # pay attention to "partial completion"
-       br      %r14
-.align 32
-.Lsoft_gmult:
-___
 $code.=<<___;
        stm${g} %r6,%r14,6*$SIZE_T($sp)
 
@@ -132,10 +113,21 @@ $code.=<<___ if(!$softonly);
                                        #  vector
        tmhh    %r0,0x4000      # check for function 65
        jz      .Lsoft_ghash
+       # Do not assume this function is called from a gcm128_context.
+       # This is not true, e.g., for AES-GCM-SIV.
+       # Parameter Block:
+       # Chaining Value (XI) 128byte
+       # Key (Htable[8]) 128byte
+       lmg     %r0,%r1,0($Xi)
+       stmg    %r0,%r1,8($sp)
+       lmg     %r0,%r1,8*16($Htbl)
+       stmg    %r0,%r1,24($sp)
+       la      %r1,8($sp)
        lghi    %r0,S390X_GHASH # function 65
-       la      %r1,0($Xi)      # H lies right after Xi in gcm128_context
        .long   0xb93e0004      # kimd %r0,$inp
        brc     1,.-4           # pay attention to "partial completion"
+       lmg     %r0,%r1,8($sp)
+       stmg    %r0,%r1,0($Xi)
        br      %r14
 .align 32
 .Lsoft_ghash: