PPC assembly pack: add ChaCha20 and Poly1305 modules.
[openssl.git] / crypto / chacha / asm / chacha-ppc.pl
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # October 2015
11
12 # ChaCha20 for PowerPC/AltiVec.
13 #
14 # Performance in cycles per byte out of large buffer.
15 #
16 #                       IALU/gcc-4.x    3xAltiVec+1xIALU
17 #
18 # Freescale e300        13.6/+115%      -
19 # PPC74x0               6.81/+310%      4.66
20 # POWER7                8.62/+61%       4.27
21 # POWER8                8.70/+51%       3.96
22
23 $flavour = shift;
24
25 if ($flavour =~ /64/) {
26         $SIZE_T =8;
27         $LRSAVE =2*$SIZE_T;
28         $STU    ="stdu";
29         $POP    ="ld";
30         $PUSH   ="std";
31         $UCMP   ="cmpld";
32 } elsif ($flavour =~ /32/) {
33         $SIZE_T =4;
34         $LRSAVE =$SIZE_T;
35         $STU    ="stwu";
36         $POP    ="lwz";
37         $PUSH   ="stw";
38         $UCMP   ="cmplw";
39 } else { die "nonsense $flavour"; }
40
41 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
42
43 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
44 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
45 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
46 die "can't locate ppc-xlate.pl";
47
48 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
49
50 $LOCALS=6*$SIZE_T;
51 $FRAME=$LOCALS+64+18*$SIZE_T;   # 64 is for local variables
52
53 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
54 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
55     $code .= "\t$opcode\t".join(',',@_)."\n";
56 }
57
58 my $sp = "r1";
59
60 my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
61
62 my @x=map("r$_",(16..31));
63 my @d=map("r$_",(11,12,14,15));
64 my @t=map("r$_",(7..10));
65
66 sub ROUND {
67 my ($a0,$b0,$c0,$d0)=@_;
68 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
69 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
70 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
71
72     (
73         "&add           (@x[$a0],@x[$a0],@x[$b0])",
74          "&add          (@x[$a1],@x[$a1],@x[$b1])",
75           "&add         (@x[$a2],@x[$a2],@x[$b2])",
76            "&add        (@x[$a3],@x[$a3],@x[$b3])",
77         "&xor           (@x[$d0],@x[$d0],@x[$a0])",
78          "&xor          (@x[$d1],@x[$d1],@x[$a1])",
79           "&xor         (@x[$d2],@x[$d2],@x[$a2])",
80            "&xor        (@x[$d3],@x[$d3],@x[$a3])",
81         "&rotlwi        (@x[$d0],@x[$d0],16)",
82          "&rotlwi       (@x[$d1],@x[$d1],16)",
83           "&rotlwi      (@x[$d2],@x[$d2],16)",
84            "&rotlwi     (@x[$d3],@x[$d3],16)",
85
86         "&add           (@x[$c0],@x[$c0],@x[$d0])",
87          "&add          (@x[$c1],@x[$c1],@x[$d1])",
88           "&add         (@x[$c2],@x[$c2],@x[$d2])",
89            "&add        (@x[$c3],@x[$c3],@x[$d3])",
90         "&xor           (@x[$b0],@x[$b0],@x[$c0])",
91          "&xor          (@x[$b1],@x[$b1],@x[$c1])",
92           "&xor         (@x[$b2],@x[$b2],@x[$c2])",
93            "&xor        (@x[$b3],@x[$b3],@x[$c3])",
94         "&rotlwi        (@x[$b0],@x[$b0],12)",
95          "&rotlwi       (@x[$b1],@x[$b1],12)",
96           "&rotlwi      (@x[$b2],@x[$b2],12)",
97            "&rotlwi     (@x[$b3],@x[$b3],12)",
98
99         "&add           (@x[$a0],@x[$a0],@x[$b0])",
100          "&add          (@x[$a1],@x[$a1],@x[$b1])",
101           "&add         (@x[$a2],@x[$a2],@x[$b2])",
102            "&add        (@x[$a3],@x[$a3],@x[$b3])",
103         "&xor           (@x[$d0],@x[$d0],@x[$a0])",
104          "&xor          (@x[$d1],@x[$d1],@x[$a1])",
105           "&xor         (@x[$d2],@x[$d2],@x[$a2])",
106            "&xor        (@x[$d3],@x[$d3],@x[$a3])",
107         "&rotlwi        (@x[$d0],@x[$d0],8)",
108          "&rotlwi       (@x[$d1],@x[$d1],8)",
109           "&rotlwi      (@x[$d2],@x[$d2],8)",
110            "&rotlwi     (@x[$d3],@x[$d3],8)",
111
112         "&add           (@x[$c0],@x[$c0],@x[$d0])",
113          "&add          (@x[$c1],@x[$c1],@x[$d1])",
114           "&add         (@x[$c2],@x[$c2],@x[$d2])",
115            "&add        (@x[$c3],@x[$c3],@x[$d3])",
116         "&xor           (@x[$b0],@x[$b0],@x[$c0])",
117          "&xor          (@x[$b1],@x[$b1],@x[$c1])",
118           "&xor         (@x[$b2],@x[$b2],@x[$c2])",
119            "&xor        (@x[$b3],@x[$b3],@x[$c3])",
120         "&rotlwi        (@x[$b0],@x[$b0],7)",
121          "&rotlwi       (@x[$b1],@x[$b1],7)",
122           "&rotlwi      (@x[$b2],@x[$b2],7)",
123            "&rotlwi     (@x[$b3],@x[$b3],7)"
124     );
125 }
126
127 $code.=<<___;
128 .machine        "any"
129
130 .globl  .ChaCha20_ctr32_int
131 .align  5
132 .ChaCha20_ctr32_int:
133 __ChaCha20_ctr32_int:
134         ${UCMP}i $len,0
135         beqlr-
136
137         $STU    $sp,-$FRAME($sp)
138         mflr    r0
139
140         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
141         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
142         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
143         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
144         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
145         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
146         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
147         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
148         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
149         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
150         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
151         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
152         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
153         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
154         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
155         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
156         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
157         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
158         $PUSH   r0,`$FRAME+$LRSAVE`($sp)
159
160         lwz     @d[0],0($ctr)                   # load counter
161         lwz     @d[1],4($ctr)
162         lwz     @d[2],8($ctr)
163         lwz     @d[3],12($ctr)
164
165         bl      __ChaCha20_1x
166
167         $POP    r0,`$FRAME+$LRSAVE`($sp)
168         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
169         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
170         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
171         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
172         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
173         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
174         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
175         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
176         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
177         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
178         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
179         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
180         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
181         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
182         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
183         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
184         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
185         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
186         mtlr    r0
187         addi    $sp,$sp,$FRAME
188         blr
189         .long   0
190         .byte   0,12,4,1,0x80,18,5,0
191         .long   0
192 .size   .ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
193
194 .align  5
195 __ChaCha20_1x:
196 Loop_outer:
197         lis     @x[0],0x6170                    # synthesize sigma
198         lis     @x[1],0x3320
199         lis     @x[2],0x7962
200         lis     @x[3],0x6b20
201         ori     @x[0],@x[0],0x7865
202         ori     @x[1],@x[1],0x646e
203         ori     @x[2],@x[2],0x2d32
204         ori     @x[3],@x[3],0x6574
205
206         li      r0,10                           # inner loop counter
207         lwz     @x[4],0($key)                   # load key
208         lwz     @x[5],4($key)
209         lwz     @x[6],8($key)
210         lwz     @x[7],12($key)
211         lwz     @x[8],16($key)
212         mr      @x[12],@d[0]                    # copy counter
213         lwz     @x[9],20($key)
214         mr      @x[13],@d[1]
215         lwz     @x[10],24($key)
216         mr      @x[14],@d[2]
217         lwz     @x[11],28($key)
218         mr      @x[15],@d[3]
219
220         mr      @t[0],@x[4]
221         mr      @t[1],@x[5]
222         mr      @t[2],@x[6]
223         mr      @t[3],@x[7]
224
225         mtctr   r0
226 Loop:
227 ___
228         foreach (&ROUND(0, 4, 8,12)) { eval; }
229         foreach (&ROUND(0, 5,10,15)) { eval; }
230 $code.=<<___;
231         bdnz    Loop
232
233         subic   $len,$len,64                    # $len-=64
234         addi    @x[0],@x[0],0x7865              # accumulate key block
235         addi    @x[1],@x[1],0x646e
236         addi    @x[2],@x[2],0x2d32
237         addi    @x[3],@x[3],0x6574
238         addis   @x[0],@x[0],0x6170
239         addis   @x[1],@x[1],0x3320
240         addis   @x[2],@x[2],0x7962
241         addis   @x[3],@x[3],0x6b20
242
243         subfe.  r0,r0,r0                        # borrow?-1:0
244         add     @x[4],@x[4],@t[0]
245         lwz     @t[0],16($key)
246         add     @x[5],@x[5],@t[1]
247         lwz     @t[1],20($key)
248         add     @x[6],@x[6],@t[2]
249         lwz     @t[2],24($key)
250         add     @x[7],@x[7],@t[3]
251         lwz     @t[3],28($key)
252         add     @x[8],@x[8],@t[0]
253         add     @x[9],@x[9],@t[1]
254         add     @x[10],@x[10],@t[2]
255         add     @x[11],@x[11],@t[3]
256
257         add     @x[12],@x[12],@d[0]
258         add     @x[13],@x[13],@d[1]
259         add     @x[14],@x[14],@d[2]
260         add     @x[15],@x[15],@d[3]
261         addi    @d[0],@d[0],1                   # increment counter
262 ___
263 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {   # flip byte order
264 $code.=<<___;
265         mr      @t[$i&3],@x[$i]
266         rotlwi  @x[$i],@x[$i],8
267         rlwimi  @x[$i],@t[$i&3],24,0,7
268         rlwimi  @x[$i],@t[$i&3],24,16,23
269 ___
270 } }
271 $code.=<<___;
272         bne     Ltail                           # $len-=64 borrowed
273
274         lwz     @t[0],0($inp)                   # load input, aligned or not
275         lwz     @t[1],4($inp)
276         ${UCMP}i $len,0                         # done already?
277         lwz     @t[2],8($inp)
278         lwz     @t[3],12($inp)
279         xor     @x[0],@x[0],@t[0]               # xor with input
280         lwz     @t[0],16($inp)
281         xor     @x[1],@x[1],@t[1]
282         lwz     @t[1],20($inp)
283         xor     @x[2],@x[2],@t[2]
284         lwz     @t[2],24($inp)
285         xor     @x[3],@x[3],@t[3]
286         lwz     @t[3],28($inp)
287         xor     @x[4],@x[4],@t[0]
288         lwz     @t[0],32($inp)
289         xor     @x[5],@x[5],@t[1]
290         lwz     @t[1],36($inp)
291         xor     @x[6],@x[6],@t[2]
292         lwz     @t[2],40($inp)
293         xor     @x[7],@x[7],@t[3]
294         lwz     @t[3],44($inp)
295         xor     @x[8],@x[8],@t[0]
296         lwz     @t[0],48($inp)
297         xor     @x[9],@x[9],@t[1]
298         lwz     @t[1],52($inp)
299         xor     @x[10],@x[10],@t[2]
300         lwz     @t[2],56($inp)
301         xor     @x[11],@x[11],@t[3]
302         lwz     @t[3],60($inp)
303         xor     @x[12],@x[12],@t[0]
304         stw     @x[0],0($out)                   # store output, aligned or not
305         xor     @x[13],@x[13],@t[1]
306         stw     @x[1],4($out)
307         xor     @x[14],@x[14],@t[2]
308         stw     @x[2],8($out)
309         xor     @x[15],@x[15],@t[3]
310         stw     @x[3],12($out)
311         stw     @x[4],16($out)
312         stw     @x[5],20($out)
313         stw     @x[6],24($out)
314         stw     @x[7],28($out)
315         stw     @x[8],32($out)
316         stw     @x[9],36($out)
317         stw     @x[10],40($out)
318         stw     @x[11],44($out)
319         stw     @x[12],48($out)
320         stw     @x[13],52($out)
321         stw     @x[14],56($out)
322         addi    $inp,$inp,64
323         stw     @x[15],60($out)
324         addi    $out,$out,64
325
326         bne     Loop_outer
327
328         blr
329
330 .align  4
331 Ltail:
332         addi    $len,$len,64                    # restore tail length
333         subi    $inp,$inp,1                     # prepare for *++ptr
334         subi    $out,$out,1
335         addi    @t[0],$sp,$LOCALS-1
336         mtctr   $len
337
338         stw     @x[0],`$LOCALS+0`($sp)          # save whole block to stack
339         stw     @x[1],`$LOCALS+4`($sp)
340         stw     @x[2],`$LOCALS+8`($sp)
341         stw     @x[3],`$LOCALS+12`($sp)
342         stw     @x[4],`$LOCALS+16`($sp)
343         stw     @x[5],`$LOCALS+20`($sp)
344         stw     @x[6],`$LOCALS+24`($sp)
345         stw     @x[7],`$LOCALS+28`($sp)
346         stw     @x[8],`$LOCALS+32`($sp)
347         stw     @x[9],`$LOCALS+36`($sp)
348         stw     @x[10],`$LOCALS+40`($sp)
349         stw     @x[11],`$LOCALS+44`($sp)
350         stw     @x[12],`$LOCALS+48`($sp)
351         stw     @x[13],`$LOCALS+52`($sp)
352         stw     @x[14],`$LOCALS+56`($sp)
353         stw     @x[15],`$LOCALS+60`($sp)
354
355 Loop_tail:                                      # byte-by-byte loop
356         lbzu    @d[0],1($inp)
357         lbzu    @x[0],1(@t[0])
358         xor     @d[1],@d[0],@x[0]
359         stbu    @d[1],1($out)
360         bdnz    Loop_tail
361
362         stw     $sp,`$LOCALS+0`($sp)            # wipe block on stack
363         stw     $sp,`$LOCALS+4`($sp)
364         stw     $sp,`$LOCALS+8`($sp)
365         stw     $sp,`$LOCALS+12`($sp)
366         stw     $sp,`$LOCALS+16`($sp)
367         stw     $sp,`$LOCALS+20`($sp)
368         stw     $sp,`$LOCALS+24`($sp)
369         stw     $sp,`$LOCALS+28`($sp)
370         stw     $sp,`$LOCALS+32`($sp)
371         stw     $sp,`$LOCALS+36`($sp)
372         stw     $sp,`$LOCALS+40`($sp)
373         stw     $sp,`$LOCALS+44`($sp)
374         stw     $sp,`$LOCALS+48`($sp)
375         stw     $sp,`$LOCALS+52`($sp)
376         stw     $sp,`$LOCALS+56`($sp)
377         stw     $sp,`$LOCALS+60`($sp)
378
379         blr
380         .long   0
381         .byte   0,12,0x14,0,0,0,0,0
382 ___
383
384 {{{
385 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2) =
386     map("v$_",(0..14));
387 my (@K)=map("v$_",(15..20));
388 my ($FOUR,$sixteen,$twenty4,$twenty,$twelve,$twenty5,$seven) =
389     map("v$_",(21..27));
390 my ($inpperm,$outperm,$outmask) = map("v$_",(28..30));
391 my @D=("v31",$seven,$T0,$T1,$T2);
392
393 my $FRAME=$LOCALS+64+13*16+18*$SIZE_T;  # 13*16 is for v20-v31 offload
394
395 sub VMXROUND {
396 my $odd = pop;
397 my ($a,$b,$c,$d,$t)=@_;
398
399         (
400         "&vadduwm       ('$a','$a','$b')",
401         "&vxor          ('$d','$d','$a')",
402         "&vperm         ('$d','$d','$d','$sixteen')",
403
404         "&vadduwm       ('$c','$c','$d')",
405         "&vxor          ('$t','$b','$c')",
406         "&vsrw          ('$b','$t','$twenty')",
407         "&vslw          ('$t','$t','$twelve')",
408         "&vor           ('$b','$b','$t')",
409
410         "&vadduwm       ('$a','$a','$b')",
411         "&vxor          ('$d','$d','$a')",
412         "&vperm         ('$d','$d','$d','$twenty4')",
413
414         "&vadduwm       ('$c','$c','$d')",
415         "&vxor          ('$t','$b','$c')",
416         "&vsrw          ('$b','$t','$twenty5')",
417         "&vslw          ('$t','$t','$seven')",
418         "&vor           ('$b','$b','$t')",
419
420         "&vsldoi        ('$c','$c','$c',8)",
421         "&vsldoi        ('$b','$b','$b',$odd?4:12)",
422         "&vsldoi        ('$d','$d','$d',$odd?12:4)"
423         );
424 }
425
426 $code.=<<___;
427
428 .globl  .ChaCha20_ctr32_vmx
429 .align  5
430 .ChaCha20_ctr32_vmx:
431         ${UCMP}i $len,256
432         blt     __ChaCha20_ctr32_int
433
434         $STU    $sp,-$FRAME($sp)
435         mflr    r0
436         li      r10,`15+$LOCALS+64`
437         li      r11,`31+$LOCALS+64`
438         mfspr   r12,256
439         stvx    v20,r10,$sp
440         addi    r10,r10,32
441         stvx    v21,r11,$sp
442         addi    r11,r11,32
443         stvx    v22,r10,$sp
444         addi    r10,r10,32
445         stvx    v23,r11,$sp
446         addi    r11,r11,32
447         stvx    v24,r10,$sp
448         addi    r10,r10,32
449         stvx    v25,r11,$sp
450         addi    r11,r11,32
451         stvx    v26,r10,$sp
452         addi    r10,r10,32
453         stvx    v27,r11,$sp
454         addi    r11,r11,32
455         stvx    v28,r10,$sp
456         addi    r10,r10,32
457         stvx    v29,r11,$sp
458         addi    r11,r11,32
459         stvx    v30,r10,$sp
460         stvx    v31,r11,$sp
461         stw     r12,`$FRAME-$SIZE_T*18-4`($sp)  # save vrsave
462         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
463         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
464         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
465         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
466         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
467         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
468         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
469         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
470         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
471         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
472         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
473         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
474         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
475         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
476         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
477         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
478         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
479         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
480         li      12,-1
481         $PUSH   r0, `$FRAME+$LRSAVE`($sp)
482         mtspr   256,r12                         # preserve all AltiVec registers
483
484         bl      Lconsts                         # returns pointer Lsigma in r12
485         li      @x[0],16
486         li      @x[1],32
487         li      @x[2],48
488         li      @x[3],64
489         li      @x[4],31                        # 31 is not a typo
490         li      @x[5],15                        # nor is 15
491
492         lvx     @K[1],0,$key                    # load key
493         ?lvsr   $T0,0,$key                      # prepare unaligned load
494         lvx     @K[2],@x[0],$key
495         lvx     @D[0],@x[4],$key
496
497         lvx     @K[3],0,$ctr                    # load counter
498         ?lvsr   $T1,0,$ctr                      # prepare unaligned load
499         lvx     @D[1],@x[5],$ctr
500
501         lvx     @K[0],0,r12                     # load constants
502         lvx     @K[5],@x[0],r12                 # one
503         lvx     $FOUR,@x[1],r12
504         lvx     $sixteen,@x[2],r12
505         lvx     $twenty4,@x[3],r12
506
507         ?vperm  @K[1],@K[2],@K[1],$T0           # align key
508         ?vperm  @K[2],@D[0],@K[2],$T0
509         ?vperm  @K[3],@D[1],@K[3],$T1           # align counter
510
511         lwz     @d[0],0($ctr)                   # load counter to GPR
512         lwz     @d[1],4($ctr)
513         vadduwm @K[3],@K[3],@K[5]               # adjust AltiVec counter
514         lwz     @d[2],8($ctr)
515         vadduwm @K[4],@K[3],@K[5]
516         lwz     @d[3],12($ctr)
517         vadduwm @K[5],@K[4],@K[5]
518
519         vspltisw $twenty,-12                    # synthesize constants 
520         vspltisw $twelve,12
521         vspltisw $twenty5,-7
522         #vspltisw $seven,7                      # synthesized in the loop
523
524         vxor    $T0,$T0,$T0                     # 0x00..00
525         vspltisw $outmask,-1                    # 0xff..ff
526         ?lvsr   $inpperm,0,$inp                 # prepare for unaligned load
527         ?lvsl   $outperm,0,$out                 # prepare for unaligned store
528         ?vperm  $outmask,$outmask,$T0,$outperm
529
530         be?vspltisb $T1,3                       # 0x03..03
531         be?vxor $inpperm,$inpperm,$T1           # swap bytes within words
532         be?vxor $outperm,$outperm,$T1
533
534         b       Loop_outer_vmx
535
536 .align  4
537 Loop_outer_vmx:
538         lis     @x[0],0x6170                    # synthesize sigma
539         lis     @x[1],0x3320
540          vmr    $A0,@K[0]
541         lis     @x[2],0x7962
542         lis     @x[3],0x6b20
543          vmr    $A1,@K[0]
544         ori     @x[0],@x[0],0x7865
545         ori     @x[1],@x[1],0x646e
546          vmr    $A2,@K[0]
547         ori     @x[2],@x[2],0x2d32
548         ori     @x[3],@x[3],0x6574
549          vmr    $B0,@K[1]
550
551         li      r0,10                           # inner loop counter
552         lwz     @x[4],0($key)                   # load key to GPR
553          vmr    $B1,@K[1]
554         lwz     @x[5],4($key)
555          vmr    $B2,@K[1]
556         lwz     @x[6],8($key)
557          vmr    $C0,@K[2]
558         lwz     @x[7],12($key)
559          vmr    $C1,@K[2]
560         lwz     @x[8],16($key)
561          vmr    $C2,@K[2]
562         mr      @x[12],@d[0]                    # copy GPR counter
563         lwz     @x[9],20($key)
564          vmr    $D0,@K[3]
565         mr      @x[13],@d[1]
566         lwz     @x[10],24($key)
567          vmr    $D1,@K[4]
568         mr      @x[14],@d[2]
569         lwz     @x[11],28($key)
570          vmr    $D2,@K[5]
571         mr      @x[15],@d[3]
572
573         mr      @t[0],@x[4]
574         mr      @t[1],@x[5]
575         mr      @t[2],@x[6]
576         mr      @t[3],@x[7]
577         vspltisw $seven,7
578
579         mtctr   r0
580         nop
581 Loop_vmx:
582 ___
583         my @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,0);
584         my @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,0);
585         my @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,0);
586         my @thread3=&ROUND(0,4,8,12);
587
588         foreach (@thread0) {
589                 eval;                   eval(shift(@thread3));
590                 eval(shift(@thread1));  eval(shift(@thread3));
591                 eval(shift(@thread2));  eval(shift(@thread3));
592         }
593
594         @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,1);
595         @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,1);
596         @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,1);
597         @thread3=&ROUND(0,5,10,15);
598
599         foreach (@thread0) {
600                 eval;                   eval(shift(@thread3));
601                 eval(shift(@thread1));  eval(shift(@thread3));
602                 eval(shift(@thread2));  eval(shift(@thread3));
603         }
604 $code.=<<___;
605         bdnz    Loop_vmx
606
607         subi    $len,$len,256                   # $len-=256
608         addi    @x[0],@x[0],0x7865              # accumulate key block
609         addi    @x[1],@x[1],0x646e
610         addi    @x[2],@x[2],0x2d32
611         addi    @x[3],@x[3],0x6574
612         addis   @x[0],@x[0],0x6170
613         addis   @x[1],@x[1],0x3320
614         addis   @x[2],@x[2],0x7962
615         addis   @x[3],@x[3],0x6b20
616         add     @x[4],@x[4],@t[0]
617         lwz     @t[0],16($key)
618         add     @x[5],@x[5],@t[1]
619         lwz     @t[1],20($key)
620         add     @x[6],@x[6],@t[2]
621         lwz     @t[2],24($key)
622         add     @x[7],@x[7],@t[3]
623         lwz     @t[3],28($key)
624         add     @x[8],@x[8],@t[0]
625         add     @x[9],@x[9],@t[1]
626         add     @x[10],@x[10],@t[2]
627         add     @x[11],@x[11],@t[3]
628         add     @x[12],@x[12],@d[0]
629         add     @x[13],@x[13],@d[1]
630         add     @x[14],@x[14],@d[2]
631         add     @x[15],@x[15],@d[3]
632
633         vadduwm $A0,$A0,@K[0]                   # accumulate key block
634         vadduwm $A1,$A1,@K[0]
635         vadduwm $A2,$A2,@K[0]
636         vadduwm $B0,$B0,@K[1]
637         vadduwm $B1,$B1,@K[1]
638         vadduwm $B2,$B2,@K[1]
639         vadduwm $C0,$C0,@K[2]
640         vadduwm $C1,$C1,@K[2]
641         vadduwm $C2,$C2,@K[2]
642         vadduwm $D0,$D0,@K[3]
643         vadduwm $D1,$D1,@K[4]
644         vadduwm $D2,$D2,@K[5]
645
646         addi    @d[0],@d[0],4                   # increment counter
647         vadduwm @K[3],@K[3],$FOUR
648         vadduwm @K[4],@K[4],$FOUR
649         vadduwm @K[5],@K[5],$FOUR
650
651 ___
652 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {   # flip byte order
653 $code.=<<___;
654         mr      @t[$i&3],@x[$i]
655         rotlwi  @x[$i],@x[$i],8
656         rlwimi  @x[$i],@t[$i&3],24,0,7
657         rlwimi  @x[$i],@t[$i&3],24,16,23
658 ___
659 } }
660 $code.=<<___;
661         lwz     @t[0],0($inp)                   # load input, aligned or not
662         lwz     @t[1],4($inp)
663         lwz     @t[2],8($inp)
664         lwz     @t[3],12($inp)
665         xor     @x[0],@x[0],@t[0]               # xor with input
666         lwz     @t[0],16($inp)
667         xor     @x[1],@x[1],@t[1]
668         lwz     @t[1],20($inp)
669         xor     @x[2],@x[2],@t[2]
670         lwz     @t[2],24($inp)
671         xor     @x[3],@x[3],@t[3]
672         lwz     @t[3],28($inp)
673         xor     @x[4],@x[4],@t[0]
674         lwz     @t[0],32($inp)
675         xor     @x[5],@x[5],@t[1]
676         lwz     @t[1],36($inp)
677         xor     @x[6],@x[6],@t[2]
678         lwz     @t[2],40($inp)
679         xor     @x[7],@x[7],@t[3]
680         lwz     @t[3],44($inp)
681         xor     @x[8],@x[8],@t[0]
682         lwz     @t[0],48($inp)
683         xor     @x[9],@x[9],@t[1]
684         lwz     @t[1],52($inp)
685         xor     @x[10],@x[10],@t[2]
686         lwz     @t[2],56($inp)
687         xor     @x[11],@x[11],@t[3]
688         lwz     @t[3],60($inp)
689         xor     @x[12],@x[12],@t[0]
690         stw     @x[0],0($out)                   # store output, aligned or not
691         xor     @x[13],@x[13],@t[1]
692         stw     @x[1],4($out)
693         xor     @x[14],@x[14],@t[2]
694         stw     @x[2],8($out)
695         xor     @x[15],@x[15],@t[3]
696         stw     @x[3],12($out)
697         addi    $inp,$inp,64
698         stw     @x[4],16($out)
699         li      @t[0],16
700         stw     @x[5],20($out)
701         li      @t[1],32
702         stw     @x[6],24($out)
703         li      @t[2],48
704         stw     @x[7],28($out)
705         li      @t[3],64
706         stw     @x[8],32($out)
707         stw     @x[9],36($out)
708         stw     @x[10],40($out)
709         stw     @x[11],44($out)
710         stw     @x[12],48($out)
711         stw     @x[13],52($out)
712         stw     @x[14],56($out)
713         stw     @x[15],60($out)
714         addi    $out,$out,64
715
716         lvx     @D[0],0,$inp                    # load input
717         lvx     @D[1],@t[0],$inp
718         lvx     @D[2],@t[1],$inp
719         lvx     @D[3],@t[2],$inp
720         lvx     @D[4],@t[3],$inp
721         addi    $inp,$inp,64
722
723         ?vperm  @D[0],@D[1],@D[0],$inpperm      # align input
724         ?vperm  @D[1],@D[2],@D[1],$inpperm
725         ?vperm  @D[2],@D[3],@D[2],$inpperm
726         ?vperm  @D[3],@D[4],@D[3],$inpperm
727         vxor    $A0,$A0,@D[0]                   # xor with input
728         vxor    $B0,$B0,@D[1]
729         lvx     @D[1],@t[0],$inp                # keep loading input
730         vxor    $C0,$C0,@D[2]
731         lvx     @D[2],@t[1],$inp
732         vxor    $D0,$D0,@D[3]
733         lvx     @D[3],@t[2],$inp
734         lvx     @D[0],@t[3],$inp
735         addi    $inp,$inp,64
736         li      @t[3],63                        # 63 is not a typo
737         vperm   $A0,$A0,$A0,$outperm            # pre-misalign output
738         vperm   $B0,$B0,$B0,$outperm
739         vperm   $C0,$C0,$C0,$outperm
740         vperm   $D0,$D0,$D0,$outperm
741
742         ?vperm  @D[4],@D[1],@D[4],$inpperm      # align input
743         ?vperm  @D[1],@D[2],@D[1],$inpperm
744         ?vperm  @D[2],@D[3],@D[2],$inpperm
745         ?vperm  @D[3],@D[0],@D[3],$inpperm
746         vxor    $A1,$A1,@D[4]
747         vxor    $B1,$B1,@D[1]
748         lvx     @D[1],@t[0],$inp                # keep loading input
749         vxor    $C1,$C1,@D[2]
750         lvx     @D[2],@t[1],$inp
751         vxor    $D1,$D1,@D[3]
752         lvx     @D[3],@t[2],$inp
753         lvx     @D[4],@t[3],$inp                # redundant in aligned case
754         addi    $inp,$inp,64
755         vperm   $A1,$A1,$A1,$outperm            # pre-misalign output
756         vperm   $B1,$B1,$B1,$outperm
757         vperm   $C1,$C1,$C1,$outperm
758         vperm   $D1,$D1,$D1,$outperm
759
760         ?vperm  @D[0],@D[1],@D[0],$inpperm      # align input
761         ?vperm  @D[1],@D[2],@D[1],$inpperm
762         ?vperm  @D[2],@D[3],@D[2],$inpperm
763         ?vperm  @D[3],@D[4],@D[3],$inpperm
764         vxor    $A2,$A2,@D[0]
765         vxor    $B2,$B2,@D[1]
766         vxor    $C2,$C2,@D[2]
767         vxor    $D2,$D2,@D[3]
768         vperm   $A2,$A2,$A2,$outperm            # pre-misalign output
769         vperm   $B2,$B2,$B2,$outperm
770         vperm   $C2,$C2,$C2,$outperm
771         vperm   $D2,$D2,$D2,$outperm
772
773         andi.   @x[1],$out,15                   # is $out aligned?
774         mr      @x[0],$out
775
776         vsel    @D[0],$A0,$B0,$outmask          # collect pre-misaligned output
777         vsel    @D[1],$B0,$C0,$outmask
778         vsel    @D[2],$C0,$D0,$outmask
779         vsel    @D[3],$D0,$A1,$outmask
780         vsel    $B0,$A1,$B1,$outmask
781         vsel    $C0,$B1,$C1,$outmask
782         vsel    $D0,$C1,$D1,$outmask
783         vsel    $A1,$D1,$A2,$outmask
784         vsel    $B1,$A2,$B2,$outmask
785         vsel    $C1,$B2,$C2,$outmask
786         vsel    $D1,$C2,$D2,$outmask
787
788         #stvx   $A0,0,$out                      # take it easy on the edges
789         stvx    @D[0],@t[0],$out                # store output
790         stvx    @D[1],@t[1],$out
791         stvx    @D[2],@t[2],$out
792         addi    $out,$out,64
793         stvx    @D[3],0,$out
794         stvx    $B0,@t[0],$out
795         stvx    $C0,@t[1],$out
796         stvx    $D0,@t[2],$out
797         addi    $out,$out,64
798         stvx    $A1,0,$out
799         stvx    $B1,@t[0],$out
800         stvx    $C1,@t[1],$out
801         stvx    $D1,@t[2],$out
802         addi    $out,$out,64
803
804         beq     Laligned_vmx
805
806         sub     @x[2],$out,@x[1]                # in misaligned case edges
807         li      @x[3],0                         # are written byte-by-byte
808 Lunaligned_tail_vmx:
809         stvebx  $D2,@x[3],@x[2]
810         addi    @x[3],@x[3],1
811         cmpw    @x[3],@x[1]
812         bne     Lunaligned_tail_vmx
813
814         sub     @x[2],@x[0],@x[1]
815 Lunaligned_head_vmx:
816         stvebx  $A0,@x[1],@x[2]
817         cmpwi   @x[1],15
818         addi    @x[1],@x[1],1
819         bne     Lunaligned_head_vmx
820
821         ${UCMP}i $len,255                       # done with 256-byte blocks yet?
822         bgt     Loop_outer_vmx
823
824         b       Ldone_vmx
825
826 .align  4
827 Laligned_vmx:
828         stvx    $A0,0,@x[0]                     # head hexaword was not stored
829
830         ${UCMP}i $len,255                       # done with 256-byte blocks yet?
831         bgt     Loop_outer_vmx
832         nop
833
834 Ldone_vmx:
835         ${UCMP}i $len,0                         # done yet?
836         bnel    __ChaCha20_1x
837
838         lwz     r12,`$FRAME-$SIZE_T*18-4`($sp)  # pull vrsave
839         li      r10,`15+$LOCALS+64`
840         li      r11,`31+$LOCALS+64`
841         mtspr   256,r12                         # restore vrsave
842         lvx     v20,r10,$sp
843         addi    r10,r10,32
844         lvx     v21,r11,$sp
845         addi    r11,r11,32
846         lvx     v22,r10,$sp
847         addi    r10,r10,32
848         lvx     v23,r11,$sp
849         addi    r11,r11,32
850         lvx     v24,r10,$sp
851         addi    r10,r10,32
852         lvx     v25,r11,$sp
853         addi    r11,r11,32
854         lvx     v26,r10,$sp
855         addi    r10,r10,32
856         lvx     v27,r11,$sp
857         addi    r11,r11,32
858         lvx     v28,r10,$sp
859         addi    r10,r10,32
860         lvx     v29,r11,$sp
861         addi    r11,r11,32
862         lvx     v30,r10,$sp
863         lvx     v31,r11,$sp
864         $POP    r0, `$FRAME+$LRSAVE`($sp)
865         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
866         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
867         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
868         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
869         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
870         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
871         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
872         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
873         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
874         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
875         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
876         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
877         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
878         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
879         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
880         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
881         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
882         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
883         mtlr    r0
884         addi    $sp,$sp,$FRAME
885         blr
886         .long   0
887         .byte   0,12,0x04,1,0x80,18,5,0
888         .long   0
889 .size   .ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
890
891 .align  5
892 Lconsts:
893         mflr    r0
894         bcl     20,31,\$+4
895         mflr    r12     #vvvvv "distance between . and _vpaes_consts
896         addi    r12,r12,`64-8`
897         mtlr    r0
898         blr
899         .long   0
900         .byte   0,12,0x14,0,0,0,0,0
901         .space  `64-9*4`
902 Lsigma:
903         .long   0x61707865,0x3320646e,0x79622d32,0x6b206574
904         .long   1,0,0,0
905         .long   4,0,0,0
906 ___
907 $code.=<<___    if ($LITTLE_ENDIAN);
908         .long   0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
909         .long   0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
910 ___
911 $code.=<<___    if (!$LITTLE_ENDIAN);   # flipped words
912         .long   0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
913         .long   0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
914 ___
915 $code.=<<___;
916 .asciz  "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
917 .align  2
918 ___
919 }}}
920
921 foreach (split("\n",$code)) {
922         s/\`([^\`]*)\`/eval $1/ge;
923
924         # instructions prefixed with '?' are endian-specific and need
925         # to be adjusted accordingly...
926         if ($flavour !~ /le$/) {        # big-endian
927             s/be\?//            or
928             s/le\?/#le#/        or
929             s/\?lvsr/lvsl/      or
930             s/\?lvsl/lvsr/      or
931             s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
932             s/(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/;
933         } else {                        # little-endian
934             s/le\?//            or
935             s/be\?/#be#/        or
936             s/\?([a-z]+)/$1/;
937         }
938
939         print $_,"\n";
940 }
941
942 close STDOUT;