X-Git-Url: https://git.openssl.org/?a=blobdiff_plain;f=engines%2Fasm%2Fe_padlock-x86_64.pl;h=f8ba1e909f3d9420bcf81c27a96411c07f4fa301;hb=46bf83f07ae1ba7fda435c90af93960e77159f4b;hp=ad61974988155beed15c27a1c27851dc98311d4e;hpb=08d62e9f1a122d2a9029a8130b55525f44274d9f;p=openssl.git

diff --git a/engines/asm/e_padlock-x86_64.pl b/engines/asm/e_padlock-x86_64.pl
index ad61974988..f8ba1e909f 100644
--- a/engines/asm/e_padlock-x86_64.pl
+++ b/engines/asm/e_padlock-x86_64.pl
@@ -23,10 +23,12 @@ $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
 ( $xlate="${dir}../../crypto/perlasm/x86_64-xlate.pl" and -f $xlate) or
 die "can't locate x86_64-xlate.pl";
 
-open STDOUT,"| $^X $xlate $flavour $output";
+open OUT,"| \"$^X\" $xlate $flavour $output";
+*STDOUT=*OUT;
 
 $code=".text\n";
 
+%PADLOCK_PREFETCH=(ecb=>128, cbc=>64, ctr32=>32);	# prefetch errata
 $PADLOCK_CHUNK=512;	# Must be a power of 2 between 32 and 2^20
 
 $ctx="%rdx";
@@ -158,6 +160,7 @@ padlock_sha1_oneshot:
 	.byte	0xf3,0x0f,0xa6,0xc8	# rep xsha1
 	movaps	(%rsp),%xmm0
 	mov	16(%rsp),%eax
+	add	\$128+8,%rsp
 	movups	%xmm0,(%rdx)		# copy-out context
 	mov	%eax,16(%rdx)
 	ret
@@ -179,6 +182,7 @@ padlock_sha1_blocks:
 	.byte	0xf3,0x0f,0xa6,0xc8	# rep xsha1
 	movaps	(%rsp),%xmm0
 	mov	16(%rsp),%eax
+	add	\$128+8,%rsp
 	movups	%xmm0,(%rdx)		# copy-out context
 	mov	%eax,16(%rdx)
 	ret
@@ -200,6 +204,7 @@ padlock_sha256_oneshot:
 	.byte	0xf3,0x0f,0xa6,0xd0	# rep xsha256
 	movaps	(%rsp),%xmm0
 	movaps	16(%rsp),%xmm1
+	add	\$128+8,%rsp
 	movups	%xmm0,(%rdx)		# copy-out context
 	movups	%xmm1,16(%rdx)
 	ret
@@ -221,6 +226,7 @@ padlock_sha256_blocks:
 	.byte	0xf3,0x0f,0xa6,0xd0	# rep xsha256
 	movaps	(%rsp),%xmm0
 	movaps	16(%rsp),%xmm1
+	add	\$128+8,%rsp
 	movups	%xmm0,(%rdx)		# copy-out context
 	movups	%xmm1,16(%rdx)
 	ret
@@ -247,6 +253,7 @@ padlock_sha512_blocks:
 	movaps	16(%rsp),%xmm1
 	movaps	32(%rsp),%xmm2
 	movaps	48(%rsp),%xmm3
+	add	\$128+8,%rsp
 	movups	%xmm0,(%rdx)		# copy-out context
 	movups	%xmm1,16(%rdx)
 	movups	%xmm2,32(%rdx)
@@ -298,16 +305,36 @@ padlock_${mode}_encrypt:
 	neg	%rax
 	and	\$$PADLOCK_CHUNK-1,$chunk	# chunk%=PADLOCK_CHUNK
 	lea	(%rax,%rbp),%rsp
+	mov	\$$PADLOCK_CHUNK,%rax
+	cmovz	%rax,$chunk			# chunk=chunk?:PADLOCK_CHUNK
 ___
 $code.=<<___				if ($mode eq "ctr32");
+.L${mode}_reenter:
 	mov	-4($ctx),%eax		# pull 32-bit counter
 	bswap	%eax
 	neg	%eax
 	and	\$`$PADLOCK_CHUNK/16-1`,%eax
-	jz	.L${mode}_loop
+	mov	\$$PADLOCK_CHUNK,$chunk
 	shl	\$4,%eax
+	cmovz	$chunk,%rax
 	cmp	%rax,$len
 	cmova	%rax,$chunk		# don't let counter cross PADLOCK_CHUNK
+	cmovbe	$len,$chunk
+___
+$code.=<<___				if ($PADLOCK_PREFETCH{$mode});
+	cmp	$chunk,$len
+	ja	.L${mode}_loop
+	mov	$inp,%rax		# check if prefetch crosses page
+	cmp	%rsp,%rbp
+	cmove	$out,%rax
+	add	$len,%rax
+	neg	%rax
+	and	\$0xfff,%rax		# distance to page boundary
+	cmp	\$$PADLOCK_PREFETCH{$mode},%rax
+	mov	\$-$PADLOCK_PREFETCH{$mode},%rax
+	cmovae	$chunk,%rax		# mask=distance<prefetch?-prefetch:-1
+	and	%rax,$chunk
+	jz	.L${mode}_unaligned_tail
 ___
 $code.=<<___;
 	jmp	.L${mode}_loop
@@ -342,12 +369,12 @@ ___
 $code.=<<___				if ($mode eq "ctr32");
 	mov	-4($ctx),%eax		# pull 32-bit counter
 	test	\$0xffff0000,%eax
-	jnz	.L${mode}_no_corr
+	jnz	.L${mode}_no_carry
 	bswap	%eax
 	add	\$0x10000,%eax
 	bswap	%eax
 	mov	%eax,-4($ctx)
-.L${mode}_no_corr:
+.L${mode}_no_carry:
 ___
 $code.=<<___;
 	mov	%r8,$out		# restore paramters
@@ -355,8 +382,8 @@ $code.=<<___;
 	test	\$0x0f,$out
 	jz	.L${mode}_out_aligned
 	mov	$chunk,$len
-	shr	\$3,$len
 	lea	(%rsp),$inp
+	shr	\$3,$len
 	.byte	0xf3,0x48,0xa5		# rep movsq
 	sub	$chunk,$out
 .L${mode}_out_aligned:
@@ -366,17 +393,62 @@ $code.=<<___;
 	add	$chunk,$inp
 	sub	$chunk,$len
 	mov	\$$PADLOCK_CHUNK,$chunk
+___
+					if (!$PADLOCK_PREFETCH{$mode}) {
+$code.=<<___;
 	jnz	.L${mode}_loop
+___
+					} else {
+$code.=<<___;
+	jz	.L${mode}_break
+	cmp	$chunk,$len
+	jae	.L${mode}_loop
+___
+$code.=<<___				if ($mode eq "ctr32");
+	mov	$len,$chunk
+	mov	$inp,%rax		# check if prefetch crosses page
+	cmp	%rsp,%rbp
+	cmove	$out,%rax
+	add	$len,%rax
+	neg	%rax
+	and	\$0xfff,%rax		# distance to page boundary
+	cmp	\$$PADLOCK_PREFETCH{$mode},%rax
+	mov	\$-$PADLOCK_PREFETCH{$mode},%rax
+	cmovae	$chunk,%rax
+	and	%rax,$chunk
+	jnz	.L${mode}_loop
+___
+$code.=<<___;
+.L${mode}_unaligned_tail:
+	xor	%eax,%eax
+	cmp	%rsp,%rbp
+	cmove	$len,%rax
+	mov	$out,%r8		# save parameters
+	mov	$len,$chunk
+	sub	%rax,%rsp		# alloca
+	shr	\$3,$len
+	lea	(%rsp),$out
+	.byte	0xf3,0x48,0xa5		# rep movsq
+	mov	%rsp,$inp
+	mov	%r8, $out		# restore parameters
+	mov	$chunk,$len
+	jmp	.L${mode}_loop
+.align	16
+.L${mode}_break:
+___
+					}
+$code.=<<___;
+	cmp	%rbp,%rsp
+	je	.L${mode}_done
 
-	test	\$0x0f,$out
-	jz	.L${mode}_done
+	pxor	%xmm0,%xmm0
+	lea	(%rsp),%rax
+.L${mode}_bzero:
+	movaps	%xmm0,(%rax)
+	lea	16(%rax),%rax
+	cmp	%rax,%rbp
+	ja	.L${mode}_bzero
 
-	mov	%rbp,$len
-	mov	%rsp,$out
-	sub	%rsp,$len
-	xor	%rax,%rax
-	shr	\$3,$len
-	.byte	0xf3,0x48,0xab		# rep stosq
 .L${mode}_done:
 	lea	(%rbp),%rsp
 	jmp	.L${mode}_exit
@@ -386,24 +458,53 @@ $code.=<<___;
 ___
 $code.=<<___				if ($mode eq "ctr32");
 	mov	-4($ctx),%eax		# pull 32-bit counter
-	mov	\$`16*0x10000`,$chunk
 	bswap	%eax
-	cmp	$len,$chunk
-	cmova	$len,$chunk
 	neg	%eax
 	and	\$0xffff,%eax
-	jz	.L${mode}_aligned_loop
+	mov	\$`16*0x10000`,$chunk
 	shl	\$4,%eax
+	cmovz	$chunk,%rax
 	cmp	%rax,$len
 	cmova	%rax,$chunk		# don't let counter cross 2^16
-	jmp	.L${mode}_aligned_loop
-.align	16
+	cmovbe	$len,$chunk
+	jbe	.L${mode}_aligned_skip
+
 .L${mode}_aligned_loop:
-	cmp	$len,$chunk
-	cmova	$len,$chunk
 	mov	$len,%r10		# save parameters
 	mov	$chunk,$len
 	mov	$chunk,%r11
+
+	lea	-16($ctx),%rax		# ivp
+	lea	16($ctx),%rbx		# key
+	shr	\$4,$len		# len/=AES_BLOCK_SIZE
+	.byte	0xf3,0x0f,0xa7,$opcode	# rep xcrypt*
+
+	mov	-4($ctx),%eax		# pull 32-bit counter
+	bswap	%eax
+	add	\$0x10000,%eax
+	bswap	%eax
+	mov	%eax,-4($ctx)
+
+	mov	%r10,$len		# restore paramters
+	sub	%r11,$len
+	mov	\$`16*0x10000`,$chunk
+	jz	.L${mode}_exit
+	cmp	$chunk,$len
+	jae	.L${mode}_aligned_loop
+
+.L${mode}_aligned_skip:
+___
+$code.=<<___				if ($PADLOCK_PREFETCH{$mode});
+	lea	($inp,$len),%rbp
+	neg	%rbp
+	and	\$0xfff,%rbp		# distance to page boundary
+	xor	%eax,%eax
+	cmp	\$$PADLOCK_PREFETCH{$mode},%rbp
+	mov	\$$PADLOCK_PREFETCH{$mode}-1,%rbp
+	cmovae	%rax,%rbp
+	and	$len,%rbp		# remainder
+	sub	%rbp,$len
+	jz	.L${mode}_aligned_tail
 ___
 $code.=<<___;
 	lea	-16($ctx),%rax		# ivp
@@ -415,18 +516,23 @@ $code.=<<___				if ($mode !~ /ecb|ctr/);
 	movdqa	(%rax),%xmm0
 	movdqa	%xmm0,-16($ctx)		# copy [or refresh] iv
 ___
-$code.=<<___				if ($mode eq "ctr32");
-	mov	-4($ctx),%eax		# pull 32-bit counter
-	bswap	%eax
-	add	\$0x10000,%eax
-	bswap	%eax
-	mov	%eax,-4($ctx)
+$code.=<<___				if ($PADLOCK_PREFETCH{$mode});
+	test	%rbp,%rbp		# check remainder
+	jz	.L${mode}_exit
 
-	mov	%r11,$chunk		# restore paramters
-	mov	%r10,$len
-	sub	$chunk,$len
-	mov	\$`16*0x10000`,$chunk
-	jnz	.L${mode}_aligned_loop
+.L${mode}_aligned_tail:
+	mov	$out,%r8
+	mov	%rbp,$chunk
+	mov	%rbp,$len
+	lea	(%rsp),%rbp
+	sub	$len,%rsp
+	shr	\$3,$len
+	lea	(%rsp),$out
+	.byte	0xf3,0x48,0xa5		# rep movsq	
+	lea	(%r8),$out
+	lea	(%rsp),$inp
+	mov	$chunk,$len
+	jmp	.L${mode}_loop
 ___
 $code.=<<___;
 .L${mode}_exit: