2 # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
19 # ChaCha20 for PowerPC/AltiVec.
21 # Performance in cycles per byte out of large buffer.
23 # IALU/gcc-4.x 3xAltiVec+1xIALU
25 # Freescale e300 13.6/+115% -
26 # PPC74x0/G4e 6.81/+310% 3.81
27 # PPC970/G5 9.29/+160% ?
28 # POWER7 8.62/+61% 3.35
29 # POWER8 8.70/+51% 2.91
30 # POWER9 8.80/+29% 4.44(*)
32 # (*) this is trade-off result, it's possible to improve it, but
33 # then it would negatively affect all others;
37 if ($flavour =~ /64/) {
44 } elsif ($flavour =~ /32/) {
51 } else { die "nonsense $flavour"; }
53 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
55 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
56 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
57 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
58 die "can't locate ppc-xlate.pl";
60 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
63 $FRAME=$LOCALS+64+18*$SIZE_T; # 64 is for local variables
65 sub AUTOLOAD() # thunk [simplified] x86-style perlasm
66 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
67 $code .= "\t$opcode\t".join(',',@_)."\n";
72 my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
74 my @x=map("r$_",(16..31));
75 my @d=map("r$_",(11,12,14,15));
76 my @t=map("r$_",(7..10));
79 my ($a0,$b0,$c0,$d0)=@_;
80 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
81 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
82 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
85 "&add (@x[$a0],@x[$a0],@x[$b0])",
86 "&add (@x[$a1],@x[$a1],@x[$b1])",
87 "&add (@x[$a2],@x[$a2],@x[$b2])",
88 "&add (@x[$a3],@x[$a3],@x[$b3])",
89 "&xor (@x[$d0],@x[$d0],@x[$a0])",
90 "&xor (@x[$d1],@x[$d1],@x[$a1])",
91 "&xor (@x[$d2],@x[$d2],@x[$a2])",
92 "&xor (@x[$d3],@x[$d3],@x[$a3])",
93 "&rotlwi (@x[$d0],@x[$d0],16)",
94 "&rotlwi (@x[$d1],@x[$d1],16)",
95 "&rotlwi (@x[$d2],@x[$d2],16)",
96 "&rotlwi (@x[$d3],@x[$d3],16)",
98 "&add (@x[$c0],@x[$c0],@x[$d0])",
99 "&add (@x[$c1],@x[$c1],@x[$d1])",
100 "&add (@x[$c2],@x[$c2],@x[$d2])",
101 "&add (@x[$c3],@x[$c3],@x[$d3])",
102 "&xor (@x[$b0],@x[$b0],@x[$c0])",
103 "&xor (@x[$b1],@x[$b1],@x[$c1])",
104 "&xor (@x[$b2],@x[$b2],@x[$c2])",
105 "&xor (@x[$b3],@x[$b3],@x[$c3])",
106 "&rotlwi (@x[$b0],@x[$b0],12)",
107 "&rotlwi (@x[$b1],@x[$b1],12)",
108 "&rotlwi (@x[$b2],@x[$b2],12)",
109 "&rotlwi (@x[$b3],@x[$b3],12)",
111 "&add (@x[$a0],@x[$a0],@x[$b0])",
112 "&add (@x[$a1],@x[$a1],@x[$b1])",
113 "&add (@x[$a2],@x[$a2],@x[$b2])",
114 "&add (@x[$a3],@x[$a3],@x[$b3])",
115 "&xor (@x[$d0],@x[$d0],@x[$a0])",
116 "&xor (@x[$d1],@x[$d1],@x[$a1])",
117 "&xor (@x[$d2],@x[$d2],@x[$a2])",
118 "&xor (@x[$d3],@x[$d3],@x[$a3])",
119 "&rotlwi (@x[$d0],@x[$d0],8)",
120 "&rotlwi (@x[$d1],@x[$d1],8)",
121 "&rotlwi (@x[$d2],@x[$d2],8)",
122 "&rotlwi (@x[$d3],@x[$d3],8)",
124 "&add (@x[$c0],@x[$c0],@x[$d0])",
125 "&add (@x[$c1],@x[$c1],@x[$d1])",
126 "&add (@x[$c2],@x[$c2],@x[$d2])",
127 "&add (@x[$c3],@x[$c3],@x[$d3])",
128 "&xor (@x[$b0],@x[$b0],@x[$c0])",
129 "&xor (@x[$b1],@x[$b1],@x[$c1])",
130 "&xor (@x[$b2],@x[$b2],@x[$c2])",
131 "&xor (@x[$b3],@x[$b3],@x[$c3])",
132 "&rotlwi (@x[$b0],@x[$b0],7)",
133 "&rotlwi (@x[$b1],@x[$b1],7)",
134 "&rotlwi (@x[$b2],@x[$b2],7)",
135 "&rotlwi (@x[$b3],@x[$b3],7)"
143 .globl .ChaCha20_ctr32_int
146 __ChaCha20_ctr32_int:
150 $STU $sp,-$FRAME($sp)
153 $PUSH r14,`$FRAME-$SIZE_T*18`($sp)
154 $PUSH r15,`$FRAME-$SIZE_T*17`($sp)
155 $PUSH r16,`$FRAME-$SIZE_T*16`($sp)
156 $PUSH r17,`$FRAME-$SIZE_T*15`($sp)
157 $PUSH r18,`$FRAME-$SIZE_T*14`($sp)
158 $PUSH r19,`$FRAME-$SIZE_T*13`($sp)
159 $PUSH r20,`$FRAME-$SIZE_T*12`($sp)
160 $PUSH r21,`$FRAME-$SIZE_T*11`($sp)
161 $PUSH r22,`$FRAME-$SIZE_T*10`($sp)
162 $PUSH r23,`$FRAME-$SIZE_T*9`($sp)
163 $PUSH r24,`$FRAME-$SIZE_T*8`($sp)
164 $PUSH r25,`$FRAME-$SIZE_T*7`($sp)
165 $PUSH r26,`$FRAME-$SIZE_T*6`($sp)
166 $PUSH r27,`$FRAME-$SIZE_T*5`($sp)
167 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
168 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
169 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
170 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
171 $PUSH r0,`$FRAME+$LRSAVE`($sp)
173 lwz @d[0],0($ctr) # load counter
180 $POP r0,`$FRAME+$LRSAVE`($sp)
181 $POP r14,`$FRAME-$SIZE_T*18`($sp)
182 $POP r15,`$FRAME-$SIZE_T*17`($sp)
183 $POP r16,`$FRAME-$SIZE_T*16`($sp)
184 $POP r17,`$FRAME-$SIZE_T*15`($sp)
185 $POP r18,`$FRAME-$SIZE_T*14`($sp)
186 $POP r19,`$FRAME-$SIZE_T*13`($sp)
187 $POP r20,`$FRAME-$SIZE_T*12`($sp)
188 $POP r21,`$FRAME-$SIZE_T*11`($sp)
189 $POP r22,`$FRAME-$SIZE_T*10`($sp)
190 $POP r23,`$FRAME-$SIZE_T*9`($sp)
191 $POP r24,`$FRAME-$SIZE_T*8`($sp)
192 $POP r25,`$FRAME-$SIZE_T*7`($sp)
193 $POP r26,`$FRAME-$SIZE_T*6`($sp)
194 $POP r27,`$FRAME-$SIZE_T*5`($sp)
195 $POP r28,`$FRAME-$SIZE_T*4`($sp)
196 $POP r29,`$FRAME-$SIZE_T*3`($sp)
197 $POP r30,`$FRAME-$SIZE_T*2`($sp)
198 $POP r31,`$FRAME-$SIZE_T*1`($sp)
203 .byte 0,12,4,1,0x80,18,5,0
205 .size .ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
210 lis @x[0],0x6170 # synthesize sigma
214 ori @x[0],@x[0],0x7865
215 ori @x[1],@x[1],0x646e
216 ori @x[2],@x[2],0x2d32
217 ori @x[3],@x[3],0x6574
219 li r0,10 # inner loop counter
220 lwz @x[4],0($key) # load key
225 mr @x[12],@d[0] # copy counter
241 foreach (&ROUND(0, 4, 8,12)) { eval; }
242 foreach (&ROUND(0, 5,10,15)) { eval; }
246 subic $len,$len,64 # $len-=64
247 addi @x[0],@x[0],0x7865 # accumulate key block
248 addi @x[1],@x[1],0x646e
249 addi @x[2],@x[2],0x2d32
250 addi @x[3],@x[3],0x6574
251 addis @x[0],@x[0],0x6170
252 addis @x[1],@x[1],0x3320
253 addis @x[2],@x[2],0x7962
254 addis @x[3],@x[3],0x6b20
256 subfe. r0,r0,r0 # borrow?-1:0
257 add @x[4],@x[4],@t[0]
259 add @x[5],@x[5],@t[1]
261 add @x[6],@x[6],@t[2]
263 add @x[7],@x[7],@t[3]
265 add @x[8],@x[8],@t[0]
266 add @x[9],@x[9],@t[1]
267 add @x[10],@x[10],@t[2]
268 add @x[11],@x[11],@t[3]
270 add @x[12],@x[12],@d[0]
271 add @x[13],@x[13],@d[1]
272 add @x[14],@x[14],@d[2]
273 add @x[15],@x[15],@d[3]
274 addi @d[0],@d[0],1 # increment counter
276 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) { # flip byte order
279 rotlwi @x[$i],@x[$i],8
280 rlwimi @x[$i],@t[$i&3],24,0,7
281 rlwimi @x[$i],@t[$i&3],24,16,23
285 bne Ltail # $len-=64 borrowed
287 lwz @t[0],0($inp) # load input, aligned or not
289 ${UCMP}i $len,0 # done already?
292 xor @x[0],@x[0],@t[0] # xor with input
294 xor @x[1],@x[1],@t[1]
296 xor @x[2],@x[2],@t[2]
298 xor @x[3],@x[3],@t[3]
300 xor @x[4],@x[4],@t[0]
302 xor @x[5],@x[5],@t[1]
304 xor @x[6],@x[6],@t[2]
306 xor @x[7],@x[7],@t[3]
308 xor @x[8],@x[8],@t[0]
310 xor @x[9],@x[9],@t[1]
312 xor @x[10],@x[10],@t[2]
314 xor @x[11],@x[11],@t[3]
316 xor @x[12],@x[12],@t[0]
317 stw @x[0],0($out) # store output, aligned or not
318 xor @x[13],@x[13],@t[1]
320 xor @x[14],@x[14],@t[2]
322 xor @x[15],@x[15],@t[3]
345 addi $len,$len,64 # restore tail length
346 subi $inp,$inp,1 # prepare for *++ptr
348 addi @t[0],$sp,$LOCALS-1
351 stw @x[0],`$LOCALS+0`($sp) # save whole block to stack
352 stw @x[1],`$LOCALS+4`($sp)
353 stw @x[2],`$LOCALS+8`($sp)
354 stw @x[3],`$LOCALS+12`($sp)
355 stw @x[4],`$LOCALS+16`($sp)
356 stw @x[5],`$LOCALS+20`($sp)
357 stw @x[6],`$LOCALS+24`($sp)
358 stw @x[7],`$LOCALS+28`($sp)
359 stw @x[8],`$LOCALS+32`($sp)
360 stw @x[9],`$LOCALS+36`($sp)
361 stw @x[10],`$LOCALS+40`($sp)
362 stw @x[11],`$LOCALS+44`($sp)
363 stw @x[12],`$LOCALS+48`($sp)
364 stw @x[13],`$LOCALS+52`($sp)
365 stw @x[14],`$LOCALS+56`($sp)
366 stw @x[15],`$LOCALS+60`($sp)
368 Loop_tail: # byte-by-byte loop
371 xor @d[1],@d[0],@x[0]
375 stw $sp,`$LOCALS+0`($sp) # wipe block on stack
376 stw $sp,`$LOCALS+4`($sp)
377 stw $sp,`$LOCALS+8`($sp)
378 stw $sp,`$LOCALS+12`($sp)
379 stw $sp,`$LOCALS+16`($sp)
380 stw $sp,`$LOCALS+20`($sp)
381 stw $sp,`$LOCALS+24`($sp)
382 stw $sp,`$LOCALS+28`($sp)
383 stw $sp,`$LOCALS+32`($sp)
384 stw $sp,`$LOCALS+36`($sp)
385 stw $sp,`$LOCALS+40`($sp)
386 stw $sp,`$LOCALS+44`($sp)
387 stw $sp,`$LOCALS+48`($sp)
388 stw $sp,`$LOCALS+52`($sp)
389 stw $sp,`$LOCALS+56`($sp)
390 stw $sp,`$LOCALS+60`($sp)
394 .byte 0,12,0x14,0,0,0,0,0
398 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2)
399 = map("v$_",(0..11));
400 my @K = map("v$_",(12..17));
401 my ($FOUR,$sixteen,$twenty4) = map("v$_",(18..19,23));
402 my ($inpperm,$outperm,$outmask) = map("v$_",(24..26));
403 my @D = map("v$_",(27..31));
404 my ($twelve,$seven,$T0,$T1) = @D;
406 my $FRAME=$LOCALS+64+10*16+18*$SIZE_T; # 10*16 is for v23-v31 offload
413 "&vadduwm ('$a','$a','$b')",
414 "&vxor ('$d','$d','$a')",
415 "&vperm ('$d','$d','$d','$sixteen')",
417 "&vadduwm ('$c','$c','$d')",
418 "&vxor ('$b','$b','$c')",
419 "&vrlw ('$b','$b','$twelve')",
421 "&vadduwm ('$a','$a','$b')",
422 "&vxor ('$d','$d','$a')",
423 "&vperm ('$d','$d','$d','$twenty4')",
425 "&vadduwm ('$c','$c','$d')",
426 "&vxor ('$b','$b','$c')",
427 "&vrlw ('$b','$b','$seven')",
429 "&vsldoi ('$c','$c','$c',8)",
430 "&vsldoi ('$b','$b','$b',$odd?4:12)",
431 "&vsldoi ('$d','$d','$d',$odd?12:4)"
437 .globl .ChaCha20_ctr32_vmx
441 blt __ChaCha20_ctr32_int
443 $STU $sp,-$FRAME($sp)
445 li r10,`15+$LOCALS+64`
446 li r11,`31+$LOCALS+64`
464 stw r12,`$FRAME-$SIZE_T*18-4`($sp) # save vrsave
465 $PUSH r14,`$FRAME-$SIZE_T*18`($sp)
466 $PUSH r15,`$FRAME-$SIZE_T*17`($sp)
467 $PUSH r16,`$FRAME-$SIZE_T*16`($sp)
468 $PUSH r17,`$FRAME-$SIZE_T*15`($sp)
469 $PUSH r18,`$FRAME-$SIZE_T*14`($sp)
470 $PUSH r19,`$FRAME-$SIZE_T*13`($sp)
471 $PUSH r20,`$FRAME-$SIZE_T*12`($sp)
472 $PUSH r21,`$FRAME-$SIZE_T*11`($sp)
473 $PUSH r22,`$FRAME-$SIZE_T*10`($sp)
474 $PUSH r23,`$FRAME-$SIZE_T*9`($sp)
475 $PUSH r24,`$FRAME-$SIZE_T*8`($sp)
476 $PUSH r25,`$FRAME-$SIZE_T*7`($sp)
477 $PUSH r26,`$FRAME-$SIZE_T*6`($sp)
478 $PUSH r27,`$FRAME-$SIZE_T*5`($sp)
479 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
480 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
481 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
482 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
484 $PUSH r0, `$FRAME+$LRSAVE`($sp)
485 mtspr 256,r12 # preserve 29 AltiVec registers
487 bl Lconsts # returns pointer Lsigma in r12
492 li @x[4],31 # 31 is not a typo
493 li @x[5],15 # nor is 15
495 lvx @K[1],0,$key # load key
496 ?lvsr $T0,0,$key # prepare unaligned load
500 lvx @K[3],0,$ctr # load counter
501 ?lvsr $T1,0,$ctr # prepare unaligned load
504 lvx @K[0],0,r12 # load constants
505 lvx @K[5],@x[0],r12 # one
507 lvx $sixteen,@x[2],r12
508 lvx $twenty4,@x[3],r12
510 ?vperm @K[1],@K[2],@K[1],$T0 # align key
511 ?vperm @K[2],@D[0],@K[2],$T0
512 ?vperm @K[3],@D[1],@K[3],$T1 # align counter
514 lwz @d[0],0($ctr) # load counter to GPR
516 vadduwm @K[3],@K[3],@K[5] # adjust AltiVec counter
518 vadduwm @K[4],@K[3],@K[5]
520 vadduwm @K[5],@K[4],@K[5]
522 vxor $T0,$T0,$T0 # 0x00..00
523 vspltisw $outmask,-1 # 0xff..ff
524 ?lvsr $inpperm,0,$inp # prepare for unaligned load
525 ?lvsl $outperm,0,$out # prepare for unaligned store
526 ?vperm $outmask,$outmask,$T0,$outperm
528 be?lvsl $T0,0,@x[0] # 0x00..0f
529 be?vspltisb $T1,3 # 0x03..03
530 be?vxor $T0,$T0,$T1 # swap bytes within words
531 be?vxor $outperm,$outperm,$T1
532 be?vperm $inpperm,$inpperm,$inpperm,$T0
534 li r0,10 # inner loop counter
539 lis @x[0],0x6170 # synthesize sigma
545 ori @x[0],@x[0],0x7865
546 ori @x[1],@x[1],0x646e
548 ori @x[2],@x[2],0x2d32
549 ori @x[3],@x[3],0x6574
552 lwz @x[4],0($key) # load key to GPR
562 mr @x[12],@d[0] # copy GPR counter
578 vspltisw $twelve,12 # synthesize constants
585 my @thread0=&VMXROUND($A0,$B0,$C0,$D0,0);
586 my @thread1=&VMXROUND($A1,$B1,$C1,$D1,0);
587 my @thread2=&VMXROUND($A2,$B2,$C2,$D2,0);
588 my @thread3=&ROUND(0,4,8,12);
592 eval(shift(@thread1));
593 eval(shift(@thread2));
595 eval(shift(@thread3));
596 eval(shift(@thread3));
597 eval(shift(@thread3));
599 foreach (@thread3) { eval; }
601 @thread0=&VMXROUND($A0,$B0,$C0,$D0,1);
602 @thread1=&VMXROUND($A1,$B1,$C1,$D1,1);
603 @thread2=&VMXROUND($A2,$B2,$C2,$D2,1);
604 @thread3=&ROUND(0,5,10,15);
608 eval(shift(@thread1));
609 eval(shift(@thread2));
611 eval(shift(@thread3));
612 eval(shift(@thread3));
613 eval(shift(@thread3));
615 foreach (@thread3) { eval; }
619 subi $len,$len,256 # $len-=256
620 addi @x[0],@x[0],0x7865 # accumulate key block
621 addi @x[1],@x[1],0x646e
622 addi @x[2],@x[2],0x2d32
623 addi @x[3],@x[3],0x6574
624 addis @x[0],@x[0],0x6170
625 addis @x[1],@x[1],0x3320
626 addis @x[2],@x[2],0x7962
627 addis @x[3],@x[3],0x6b20
628 add @x[4],@x[4],@t[0]
630 add @x[5],@x[5],@t[1]
632 add @x[6],@x[6],@t[2]
634 add @x[7],@x[7],@t[3]
636 add @x[8],@x[8],@t[0]
637 add @x[9],@x[9],@t[1]
638 add @x[10],@x[10],@t[2]
639 add @x[11],@x[11],@t[3]
640 add @x[12],@x[12],@d[0]
641 add @x[13],@x[13],@d[1]
642 add @x[14],@x[14],@d[2]
643 add @x[15],@x[15],@d[3]
645 vadduwm $A0,$A0,@K[0] # accumulate key block
646 vadduwm $A1,$A1,@K[0]
647 vadduwm $A2,$A2,@K[0]
648 vadduwm $B0,$B0,@K[1]
649 vadduwm $B1,$B1,@K[1]
650 vadduwm $B2,$B2,@K[1]
651 vadduwm $C0,$C0,@K[2]
652 vadduwm $C1,$C1,@K[2]
653 vadduwm $C2,$C2,@K[2]
654 vadduwm $D0,$D0,@K[3]
655 vadduwm $D1,$D1,@K[4]
656 vadduwm $D2,$D2,@K[5]
658 addi @d[0],@d[0],4 # increment counter
659 vadduwm @K[3],@K[3],$FOUR
660 vadduwm @K[4],@K[4],$FOUR
661 vadduwm @K[5],@K[5],$FOUR
664 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) { # flip byte order
667 rotlwi @x[$i],@x[$i],8
668 rlwimi @x[$i],@t[$i&3],24,0,7
669 rlwimi @x[$i],@t[$i&3],24,16,23
673 lwz @t[0],0($inp) # load input, aligned or not
677 xor @x[0],@x[0],@t[0] # xor with input
679 xor @x[1],@x[1],@t[1]
681 xor @x[2],@x[2],@t[2]
683 xor @x[3],@x[3],@t[3]
685 xor @x[4],@x[4],@t[0]
687 xor @x[5],@x[5],@t[1]
689 xor @x[6],@x[6],@t[2]
691 xor @x[7],@x[7],@t[3]
693 xor @x[8],@x[8],@t[0]
695 xor @x[9],@x[9],@t[1]
697 xor @x[10],@x[10],@t[2]
699 xor @x[11],@x[11],@t[3]
701 xor @x[12],@x[12],@t[0]
702 stw @x[0],0($out) # store output, aligned or not
703 xor @x[13],@x[13],@t[1]
705 xor @x[14],@x[14],@t[2]
707 xor @x[15],@x[15],@t[3]
728 lvx @D[0],0,$inp # load input
735 ?vperm @D[0],@D[1],@D[0],$inpperm # align input
736 ?vperm @D[1],@D[2],@D[1],$inpperm
737 ?vperm @D[2],@D[3],@D[2],$inpperm
738 ?vperm @D[3],@D[4],@D[3],$inpperm
739 vxor $A0,$A0,@D[0] # xor with input
741 lvx @D[1],@t[0],$inp # keep loading input
748 li @t[3],63 # 63 is not a typo
749 vperm $A0,$A0,$A0,$outperm # pre-misalign output
750 vperm $B0,$B0,$B0,$outperm
751 vperm $C0,$C0,$C0,$outperm
752 vperm $D0,$D0,$D0,$outperm
754 ?vperm @D[4],@D[1],@D[4],$inpperm # align input
755 ?vperm @D[1],@D[2],@D[1],$inpperm
756 ?vperm @D[2],@D[3],@D[2],$inpperm
757 ?vperm @D[3],@D[0],@D[3],$inpperm
760 lvx @D[1],@t[0],$inp # keep loading input
765 lvx @D[4],@t[3],$inp # redundant in aligned case
767 vperm $A1,$A1,$A1,$outperm # pre-misalign output
768 vperm $B1,$B1,$B1,$outperm
769 vperm $C1,$C1,$C1,$outperm
770 vperm $D1,$D1,$D1,$outperm
772 ?vperm @D[0],@D[1],@D[0],$inpperm # align input
773 ?vperm @D[1],@D[2],@D[1],$inpperm
774 ?vperm @D[2],@D[3],@D[2],$inpperm
775 ?vperm @D[3],@D[4],@D[3],$inpperm
780 vperm $A2,$A2,$A2,$outperm # pre-misalign output
781 vperm $B2,$B2,$B2,$outperm
782 vperm $C2,$C2,$C2,$outperm
783 vperm $D2,$D2,$D2,$outperm
785 andi. @x[1],$out,15 # is $out aligned?
788 vsel @D[0],$A0,$B0,$outmask # collect pre-misaligned output
789 vsel @D[1],$B0,$C0,$outmask
790 vsel @D[2],$C0,$D0,$outmask
791 vsel @D[3],$D0,$A1,$outmask
792 vsel $B0,$A1,$B1,$outmask
793 vsel $C0,$B1,$C1,$outmask
794 vsel $D0,$C1,$D1,$outmask
795 vsel $A1,$D1,$A2,$outmask
796 vsel $B1,$A2,$B2,$outmask
797 vsel $C1,$B2,$C2,$outmask
798 vsel $D1,$C2,$D2,$outmask
800 #stvx $A0,0,$out # take it easy on the edges
801 stvx @D[0],@t[0],$out # store output
802 stvx @D[1],@t[1],$out
803 stvx @D[2],@t[2],$out
818 sub @x[2],$out,@x[1] # in misaligned case edges
819 li @x[3],0 # are written byte-by-byte
821 stvebx $D2,@x[3],@x[2]
824 bne Lunaligned_tail_vmx
826 sub @x[2],@x[0],@x[1]
828 stvebx $A0,@x[1],@x[2]
831 bne Lunaligned_head_vmx
833 ${UCMP}i $len,255 # done with 256-byte blocks yet?
840 stvx $A0,0,@x[0] # head hexaword was not stored
842 ${UCMP}i $len,255 # done with 256-byte blocks yet?
847 ${UCMP}i $len,0 # done yet?
850 lwz r12,`$FRAME-$SIZE_T*18-4`($sp) # pull vrsave
851 li r10,`15+$LOCALS+64`
852 li r11,`31+$LOCALS+64`
853 mtspr 256,r12 # restore vrsave
870 $POP r0, `$FRAME+$LRSAVE`($sp)
871 $POP r14,`$FRAME-$SIZE_T*18`($sp)
872 $POP r15,`$FRAME-$SIZE_T*17`($sp)
873 $POP r16,`$FRAME-$SIZE_T*16`($sp)
874 $POP r17,`$FRAME-$SIZE_T*15`($sp)
875 $POP r18,`$FRAME-$SIZE_T*14`($sp)
876 $POP r19,`$FRAME-$SIZE_T*13`($sp)
877 $POP r20,`$FRAME-$SIZE_T*12`($sp)
878 $POP r21,`$FRAME-$SIZE_T*11`($sp)
879 $POP r22,`$FRAME-$SIZE_T*10`($sp)
880 $POP r23,`$FRAME-$SIZE_T*9`($sp)
881 $POP r24,`$FRAME-$SIZE_T*8`($sp)
882 $POP r25,`$FRAME-$SIZE_T*7`($sp)
883 $POP r26,`$FRAME-$SIZE_T*6`($sp)
884 $POP r27,`$FRAME-$SIZE_T*5`($sp)
885 $POP r28,`$FRAME-$SIZE_T*4`($sp)
886 $POP r29,`$FRAME-$SIZE_T*3`($sp)
887 $POP r30,`$FRAME-$SIZE_T*2`($sp)
888 $POP r31,`$FRAME-$SIZE_T*1`($sp)
893 .byte 0,12,0x04,1,0x80,18,5,0
895 .size .ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
901 mflr r12 #vvvvv "distance between . and Lsigma
906 .byte 0,12,0x14,0,0,0,0,0
909 .long 0x61707865,0x3320646e,0x79622d32,0x6b206574
913 $code.=<<___ if ($LITTLE_ENDIAN);
914 .long 0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
915 .long 0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
917 $code.=<<___ if (!$LITTLE_ENDIAN); # flipped words
918 .long 0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
919 .long 0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
922 .asciz "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
927 foreach (split("\n",$code)) {
928 s/\`([^\`]*)\`/eval $1/ge;
930 # instructions prefixed with '?' are endian-specific and need
931 # to be adjusted accordingly...
932 if ($flavour !~ /le$/) { # big-endian
937 s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
938 s/(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/;
939 } else { # little-endian