Run util/openssl-format-source on the Curve448 code
[openssl.git] / crypto / ec / curve448 / arch_neon / f_impl.c
1 /*
2  * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
3  * Copyright 2014 Cryptography Research, Inc.
4  *
5  * Licensed under the OpenSSL license (the "License").  You may not use
6  * this file except in compliance with the License.  You can obtain a copy
7  * in the file LICENSE in the source distribution or at
8  * https://www.openssl.org/source/license.html
9  *
10  * Originally written by Mike Hamburg
11  */
12
13 #include "f_field.h"
14
15 static __inline__ uint64x2_t __attribute__ ((gnu_inline, always_inline, unused))
16     xx_vaddup_u64(uint64x2_t x)
17 {
18  __asm__("vadd.s64 %f0, %e0":"+w"(x));
19     return x;
20 }
21
22 static __inline__ int64x2_t __attribute__ ((gnu_inline, always_inline, unused))
23     vrev128_s64(int64x2_t x)
24 {
25  __asm__("vswp.s64 %e0, %f0":"+w"(x));
26     return x;
27 }
28
29 static __inline__ uint64x2_t __attribute__ ((gnu_inline, always_inline))
30     vrev128_u64(uint64x2_t x)
31 {
32  __asm__("vswp.s64 %e0, %f0":"+w"(x));
33     return x;
34 }
35
36 static inline void __attribute__ ((gnu_inline, always_inline, unused))
37     smlal(uint64_t *acc, const uint32_t a, const uint32_t b)
38 {
39     *acc += (int64_t)(int32_t)a *(int64_t)(int32_t)b;
40 }
41
42 static inline void __attribute__ ((gnu_inline, always_inline, unused))
43     smlal2(uint64_t *acc, const uint32_t a, const uint32_t b)
44 {
45     *acc += (int64_t)(int32_t)a *(int64_t)(int32_t)b *2;
46 }
47
48 static inline void __attribute__ ((gnu_inline, always_inline, unused))
49     smull(uint64_t *acc, const uint32_t a, const uint32_t b)
50 {
51     *acc = (int64_t)(int32_t)a *(int64_t)(int32_t)b;
52 }
53
54 static inline void __attribute__ ((gnu_inline, always_inline, unused))
55     smull2(uint64_t *acc, const uint32_t a, const uint32_t b)
56 {
57     *acc = (int64_t)(int32_t)a *(int64_t)(int32_t)b *2;
58 }
59
60 void gf_mul(gf_s * __restrict__ cs, const gf as, const gf bs)
61 {
62 #define _bl0 "q0"
63 #define _bl0_0 "d0"
64 #define _bl0_1 "d1"
65 #define _bh0 "q1"
66 #define _bh0_0 "d2"
67 #define _bh0_1 "d3"
68 #define _bs0 "q2"
69 #define _bs0_0 "d4"
70 #define _bs0_1 "d5"
71 #define _bl2 "q3"
72 #define _bl2_0 "d6"
73 #define _bl2_1 "d7"
74 #define _bh2 "q4"
75 #define _bh2_0 "d8"
76 #define _bh2_1 "d9"
77 #define _bs2 "q5"
78 #define _bs2_0 "d10"
79 #define _bs2_1 "d11"
80
81 #define _as0 "q6"
82 #define _as0_0 "d12"
83 #define _as0_1 "d13"
84 #define _as2 "q7"
85 #define _as2_0 "d14"
86 #define _as2_1 "d15"
87 #define _al0 "q8"
88 #define _al0_0 "d16"
89 #define _al0_1 "d17"
90 #define _ah0 "q9"
91 #define _ah0_0 "d18"
92 #define _ah0_1 "d19"
93 #define _al2 "q10"
94 #define _al2_0 "d20"
95 #define _al2_1 "d21"
96 #define _ah2 "q11"
97 #define _ah2_0 "d22"
98 #define _ah2_1 "d23"
99
100 #define _a0a "q12"
101 #define _a0a_0 "d24"
102 #define _a0a_1 "d25"
103 #define _a0b "q13"
104 #define _a0b_0 "d26"
105 #define _a0b_1 "d27"
106 #define _a1a "q14"
107 #define _a1a_0 "d28"
108 #define _a1a_1 "d29"
109 #define _a1b "q15"
110 #define _a1b_0 "d30"
111 #define _a1b_1 "d31"
112 #define VMAC(op,result,a,b,n) #op" "result", "a", "b"[" #n "]\n\t"
113 #define VOP3(op,result,a,b)   #op" "result", "a", "b"\n\t"
114 #define VOP2(op,result,a)     #op" "result", "a"\n\t"
115
116     int32x2_t *vc = (int32x2_t *) cs->limb;
117
118     __asm__ __volatile__("vld2.32 {" _al0_0 "," _al0_1 "," _ah0_0 "," _ah0_1
119                          "}, [%[a],:128]!" "\n\t" VOP3(vadd.i32, _as0, _al0,
120                                                        _ah0)
121                          "vld2.32 {" _bl0_0 "," _bl0_1 "," _bh0_0 "," _bh0_1
122                          "}, [%[b],:128]!" "\n\t" VOP3(vadd.i32, _bs0_1, _bl0_1,
123                                                        _bh0_1) VOP3(vsub.i32,
124                                                                     _bs0_0,
125                                                                     _bl0_0,
126                                                                     _bh0_0)
127                          "vld2.32 {" _bl2_0 "," _bl2_1 "," _bh2_0 "," _bh2_1
128                          "}, [%[b],:128]!" "\n\t" VOP3(vadd.i32, _bs2, _bl2,
129                                                        _bh2)
130                          "vld2.32 {" _al2_0 "," _al2_1 "," _ah2_0 "," _ah2_1
131                          "}, [%[a],:128]!" "\n\t" VOP3(vadd.i32, _as2, _al2,
132                                                        _ah2)
133                          VMAC(vmull.s32, _a0b, _as0_1, _bs2_1,
134                               0) VMAC(vmlal.s32, _a0b, _as2_0, _bs2_0,
135                                       0) VMAC(vmlal.s32, _a0b, _as2_1, _bs0_1,
136                                               0) VMAC(vmlal.s32, _a0b, _as0_0,
137                                                       _bh0_0, 0)
138                          VMAC(vmull.s32, _a1b, _as0_1, _bs2_1,
139                               1) VMAC(vmlal.s32, _a1b, _as2_0, _bs2_0,
140                                       1) VMAC(vmlal.s32, _a1b, _as2_1, _bs0_1,
141                                               1) VMAC(vmlal.s32, _a1b, _as0_0,
142                                                       _bh0_0, 1)
143                          VOP2(vmov, _a0a, _a0b) VMAC(vmlal.s32, _a0a, _ah0_1,
144                                                      _bh2_1, 0) VMAC(vmlal.s32,
145                                                                      _a0a,
146                                                                      _ah2_0,
147                                                                      _bh2_0,
148                                                                      0)
149                          VMAC(vmlal.s32, _a0a, _ah2_1, _bh0_1,
150                               0) VMAC(vmlal.s32, _a0a, _ah0_0, _bl0_0, 0)
151                          VMAC(vmlsl.s32, _a0b, _al0_1, _bl2_1,
152                               0) VMAC(vmlsl.s32, _a0b, _al2_0, _bl2_0,
153                                       0) VMAC(vmlsl.s32, _a0b, _al2_1, _bl0_1,
154                                               0) VMAC(vmlal.s32, _a0b, _al0_0,
155                                                       _bs0_0, 0)
156                          VOP2(vmov, _a1a, _a1b) VMAC(vmlal.s32, _a1a, _ah0_1,
157                                                      _bh2_1, 1) VMAC(vmlal.s32,
158                                                                      _a1a,
159                                                                      _ah2_0,
160                                                                      _bh2_0,
161                                                                      1)
162                          VMAC(vmlal.s32, _a1a, _ah2_1, _bh0_1,
163                               1) VMAC(vmlal.s32, _a1a, _ah0_0, _bl0_0, 1)
164                          VOP2(vswp, _a0b_1, _a0a_0)
165                          VMAC(vmlsl.s32, _a1b, _al0_1, _bl2_1, 1)
166                          VMAC(vmlsl.s32, _a1b, _al2_0, _bl2_0, 1)
167                          VMAC(vmlsl.s32, _a1b, _al2_1, _bl0_1, 1)
168                          VMAC(vmlal.s32, _a1b, _al0_0, _bs0_0, 1)
169                          VOP3(vsra.u64, _a0a, _a0b, "#28")
170                          VOP3(vsub.i32, _bs0_1, _bl0_1, _bh0_1)
171                          VOP2(vmovn.i64, _a0b_0, _a0b)
172                          VOP2(vswp, _a1b_1, _a1a_0)
173                          VOP3(vadd.i64, _a1b, _a0a, _a1b)
174                          VMAC(vmull.s32, _a0a, _as2_0, _bs2_1, 0)
175                          VOP2(vmovn.i64, _a0b_1, _a1b)
176                          VMAC(vmlal.s32, _a0a, _as2_1, _bs2_0, 0)
177                          VOP3(vsra.u64, _a1a, _a1b, "#28")
178                          VMAC(vmlal.s32, _a0a, _as0_0, _bh0_1, 0)
179                          VOP2(vbic.i32, _a0b, "#0xf0000000")
180                          VMAC(vmlal.s32, _a0a, _as0_1, _bh0_0, 0)
181                          "vstmia %[c]!, {" _a0b_0 ", " _a0b_1 "}" "\n\t"
182                          VMAC(vmull.s32, _a1b, _as2_0, _bs2_1, 1)
183                          VMAC(vmlal.s32, _a1b, _as2_1, _bs2_0, 1)
184                          VMAC(vmlal.s32, _a1b, _as0_0, _bh0_1, 1)
185                          VMAC(vmlal.s32, _a1b, _as0_1, _bh0_0, 1)
186                          VOP2(vmov, _a0b_1, _a0a_1)
187                          VOP3(vadd.i64, _a0b_0, _a0a_0, _a1a_0)
188                          VOP3(vadd.i64, _a0a_0, _a0a_0, _a1a_1)
189                          VMAC(vmlal.s32, _a0a, _ah2_0, _bh2_1, 0)
190                          VMAC(vmlal.s32, _a0a, _ah2_1, _bh2_0, 0)
191                          VMAC(vmlal.s32, _a0a, _ah0_0, _bl0_1, 0)
192                          VMAC(vmlal.s32, _a0a, _ah0_1, _bl0_0, 0)
193                          VMAC(vmlsl.s32, _a0b, _al2_0, _bl2_1, 0)
194                          VMAC(vmlsl.s32, _a0b, _al2_1, _bl2_0, 0)
195                          VMAC(vmlal.s32, _a0b, _al0_0, _bs0_1, 0)
196                          VMAC(vmlal.s32, _a0b, _al0_1, _bs0_0, 0)
197                          VOP2(vmov, _a1a, _a1b)
198                          VMAC(vmlal.s32, _a1a, _ah2_0, _bh2_1, 1)
199                          VMAC(vmlal.s32, _a1a, _ah2_1, _bh2_0, 1)
200                          VMAC(vmlal.s32, _a1a, _ah0_0, _bl0_1, 1)
201                          VMAC(vmlal.s32, _a1a, _ah0_1, _bl0_0, 1)
202                          VOP2(vswp, _a0b_1, _a0a_0)
203                          VMAC(vmlsl.s32, _a1b, _al2_0, _bl2_1, 1)
204                          VMAC(vmlsl.s32, _a1b, _al2_1, _bl2_0, 1)
205                          VMAC(vmlal.s32, _a1b, _al0_0, _bs0_1, 1)
206                          VMAC(vmlal.s32, _a1b, _al0_1, _bs0_0, 1)
207                          VOP3(vsra.u64, _a0a, _a0b, "#28")
208                          VOP3(vsub.i32, _bs2_0, _bl2_0, _bh2_0)
209                          VOP2(vmovn.i64, _a0b_0, _a0b)
210                          VOP2(vswp, _a1b_1, _a1a_0)
211                          VOP3(vadd.i64, _a1b, _a0a, _a1b)
212                          VMAC(vmull.s32, _a0a, _as2_1, _bs2_1, 0)
213                          VOP2(vmovn.i64, _a0b_1, _a1b)
214                          VMAC(vmlal.s32, _a0a, _as0_0, _bh2_0, 0)
215                          VOP3(vsra.u64, _a1a, _a1b, "#28")
216                          VMAC(vmlal.s32, _a0a, _as0_1, _bh0_1, 0)
217                          VOP2(vbic.i32, _a0b, "#0xf0000000")
218                          VMAC(vmlal.s32, _a0a, _as2_0, _bh0_0, 0)
219                          "vstmia %[c]!, {" _a0b_0 ", " _a0b_1 "}" "\n\t"
220                          VMAC(vmull.s32, _a1b, _as2_1, _bs2_1, 1)
221                          VMAC(vmlal.s32, _a1b, _as0_0, _bh2_0, 1)
222                          VMAC(vmlal.s32, _a1b, _as0_1, _bh0_1, 1)
223                          VMAC(vmlal.s32, _a1b, _as2_0, _bh0_0, 1)
224                          VOP2(vmov, _a0b_1, _a0a_1)
225                          VOP3(vadd.i64, _a0b_0, _a0a_0, _a1a_0)
226                          VOP3(vadd.i64, _a0a_0, _a0a_0, _a1a_1)
227                          VMAC(vmlal.s32, _a0a, _ah2_1, _bh2_1, 0)
228                          VMAC(vmlal.s32, _a0a, _ah0_0, _bl2_0, 0)
229                          VMAC(vmlal.s32, _a0a, _ah0_1, _bl0_1, 0)
230                          VMAC(vmlal.s32, _a0a, _ah2_0, _bl0_0, 0)
231                          VMAC(vmlsl.s32, _a0b, _al2_1, _bl2_1, 0)
232                          VMAC(vmlal.s32, _a0b, _al0_0, _bs2_0, 0)
233                          VMAC(vmlal.s32, _a0b, _al0_1, _bs0_1, 0)
234                          VMAC(vmlal.s32, _a0b, _al2_0, _bs0_0, 0)
235                          VOP2(vmov, _a1a, _a1b)
236                          VMAC(vmlal.s32, _a1a, _ah2_1, _bh2_1, 1)
237                          VMAC(vmlal.s32, _a1a, _ah0_0, _bl2_0, 1)
238                          VMAC(vmlal.s32, _a1a, _ah0_1, _bl0_1, 1)
239                          VMAC(vmlal.s32, _a1a, _ah2_0, _bl0_0, 1)
240                          VOP2(vswp, _a0b_1, _a0a_0)
241                          VMAC(vmlsl.s32, _a1b, _al2_1, _bl2_1, 1)
242                          VMAC(vmlal.s32, _a1b, _al0_0, _bs2_0, 1)
243                          VMAC(vmlal.s32, _a1b, _al0_1, _bs0_1, 1)
244                          VMAC(vmlal.s32, _a1b, _al2_0, _bs0_0, 1)
245                          VOP3(vsub.i32, _bs2_1, _bl2_1, _bh2_1)
246                          VOP3(vsra.u64, _a0a, _a0b, "#28")
247                          VOP2(vmovn.i64, _a0b_0, _a0b)
248                          VOP2(vswp, _a1b_1, _a1a_0)
249                          VOP3(vadd.i64, _a1b, _a0a, _a1b)
250                          VMAC(vmull.s32, _a0a, _as0_0, _bh2_1, 0)
251                          VOP2(vmovn.i64, _a0b_1, _a1b)
252                          VMAC(vmlal.s32, _a0a, _as0_1, _bh2_0, 0)
253                          VOP3(vsra.u64, _a1a, _a1b, "#28")
254                          VMAC(vmlal.s32, _a0a, _as2_0, _bh0_1, 0)
255                          VOP2(vbic.i32, _a0b, "#0xf0000000")
256                          VMAC(vmlal.s32, _a0a, _as2_1, _bh0_0, 0)
257                          "vstmia %[c]!, {" _a0b_0 ", " _a0b_1 "}" "\n\t"
258                          VMAC(vmull.s32, _a1b, _as0_0, _bh2_1, 1)
259                          VMAC(vmlal.s32, _a1b, _as0_1, _bh2_0, 1)
260                          VMAC(vmlal.s32, _a1b, _as2_0, _bh0_1, 1)
261                          VMAC(vmlal.s32, _a1b, _as2_1, _bh0_0, 1)
262                          VOP2(vmov, _a0b_1, _a0a_1)
263                          VOP3(vadd.i64, _a0b_0, _a0a_0, _a1a_0)
264                          VOP3(vadd.i64, _a0a_0, _a0a_0, _a1a_1)
265                          VMAC(vmlal.s32, _a0a, _ah0_0, _bl2_1, 0)
266                          VMAC(vmlal.s32, _a0a, _ah0_1, _bl2_0, 0)
267                          VMAC(vmlal.s32, _a0a, _ah2_0, _bl0_1, 0)
268                          VMAC(vmlal.s32, _a0a, _ah2_1, _bl0_0, 0)
269                          VMAC(vmlal.s32, _a0b, _al0_0, _bs2_1, 0)
270                          VMAC(vmlal.s32, _a0b, _al0_1, _bs2_0, 0)
271                          VMAC(vmlal.s32, _a0b, _al2_0, _bs0_1, 0)
272                          VMAC(vmlal.s32, _a0b, _al2_1, _bs0_0, 0)
273                          VOP2(vmov, _a1a, _a1b)
274                          VMAC(vmlal.s32, _a1a, _ah0_0, _bl2_1, 1)
275                          VMAC(vmlal.s32, _a1a, _ah0_1, _bl2_0, 1)
276                          VMAC(vmlal.s32, _a1a, _ah2_0, _bl0_1, 1)
277                          VMAC(vmlal.s32, _a1a, _ah2_1, _bl0_0, 1)
278                          VOP2(vswp, _a0b_1, _a0a_0)
279                          VMAC(vmlal.s32, _a1b, _al0_0, _bs2_1, 1)
280                          VMAC(vmlal.s32, _a1b, _al0_1, _bs2_0, 1)
281                          VMAC(vmlal.s32, _a1b, _al2_0, _bs0_1, 1)
282                          VMAC(vmlal.s32, _a1b, _al2_1, _bs0_0, 1)
283                          VOP3(vsra.u64, _a0a, _a0b, "#28")
284                          VOP2(vmovn.i64, _a0b_0, _a0b)
285                          VOP2(vswp, _a1b_1, _a1a_0)
286                          VOP3(vadd.i64, _a0a, _a0a, _a1b)
287                          VOP2(vmovn.i64, _a0b_1, _a0a)
288                          VOP3(vsra.u64, _a1a, _a0a, "#28")
289                          VOP2(vbic.i32, _a0b, "#0xf0000000")
290                          VOP2(vswp, _a1a_0, _a1a_1)
291                          "vstmia %[c]!, {" _a0b_0 ", " _a0b_1 "}" "\n\t"
292                          "sub %[c], #64" "\n\t"
293                          VOP3(vadd.i64, _a1a_1, _a1a_1, _a1a_0)
294                          "vldmia %[c], {" _a0a_0 ", " _a0a_1 ", " _a0b_0 "}"
295                          "\n\t" VOP2(vaddw.s32, _a1a, _a0a_0) VOP2(vmovn.i64,
296                                                                    _a0a_0,
297                                                                    _a1a)
298                          VOP2(vshr.s64, _a1a, "#28")
299                          VOP2(vaddw.s32, _a1a, _a0a_1) VOP2(vmovn.i64, _a0a_1,
300                                                             _a1a) VOP2(vshr.s64,
301                                                                        _a1a,
302                                                                        "#28")
303                          VOP2(vbic.i32, _a0a, "#0xf0000000")
304                          VOP2(vaddw.s32, _a1a, _a0b_0)
305                          VOP2(vmovn.i64, _a0b_0, _a1a)
306                          "vstmia %[c], {" _a0a_0 ", " _a0a_1 ", " _a0b_0 "}"
307                          "\n\t":[a] "+r"(as)
308                          ,[b] "+r"(bs)
309                          ,[c] "+r"(vc)
310
311                          ::"q0", "q1", "q2", "q3",
312                          "q4", "q5", "q6", "q7",
313                          "q8", "q9", "q10", "q11",
314                          "q12", "q13", "q14", "q15", "memory");
315 }
316
317 void gf_sqr(gf_s * __restrict__ cs, const gf bs)
318 {
319     int32x2_t *vc = (int32x2_t *) cs->limb;
320
321     __asm__ __volatile__("vld2.32 {" _bl0_0 "," _bl0_1 "," _bh0_0 "," _bh0_1 "}, [%[b],:128]!" "\n\t" VOP3(vadd.i32, _bs0_1, _bl0_1, _bh0_1) /* 0
322                                                                                                                                               * *
323                                                                                                                                               * ..
324                                                                                                                                               * *
325                                                                                                                                               * 2^30 */ VOP3(vsub.i32, _bs0_0, _bl0_0, _bh0_0)
326                          /* +- 2^29 */
327                          VOP3(vadd.i32, _as0, _bl0, _bh0)
328                          /*
329                           * 0 .. 2^30
330                           */
331                          "vld2.32 {" _bl2_0 "," _bl2_1 "," _bh2_0 "," _bh2_1 "}, [%[b],:128]!" "\n\t" VOP3(vadd.i32, _bs2, _bl2, _bh2) /* 0
332                                                                                                                                         * *
333                                                                                                                                         * ..
334                                                                                                                                         * *
335                                                                                                                                         * 2^30 */
336                          VOP2(vmov, _as2, _bs2)
337
338                          VMAC(vqdmull.s32, _a0b, _as0_1, _bs2_1, 0) /* 0 .. 8 *
339                                                                      * * 2^58. *
340                                                                      * danger *
341                                                                      * for *
342                                                                      * vqdmlal *
343                                                                      * is 32 */
344                          VMAC(vmlal.s32, _a0b, _as2_0, _bs2_0, 0) /* 0 .. 12 */
345                          VMAC(vmlal.s32, _a0b, _as0_0, _bh0_0, 0)
346                          /*
347                           * 0 .. 14
348                           */
349                          VMAC(vqdmull.s32, _a1b, _as0_1, _bs2_1, 1) /* 0 .. 8 */
350                          VMAC(vmlal.s32, _a1b, _as2_0, _bs2_0, 1) /* 0 .. 14 */
351                          VMAC(vmlal.s32, _a1b, _as0_0, _bh0_0, 1)
352                          /*
353                           * 0 .. 16
354                           */
355                          VOP2(vmov, _a0a, _a0b) /* 0 .. 14 */
356                          VMAC(vqdmlal.s32, _a0a, _bh0_1, _bh2_1, 0) /* 0 .. 16 */
357                          VMAC(vmlal.s32, _a0a, _bh2_0, _bh2_0, 0) /* 0 .. 17 */
358                          VMAC(vmlal.s32, _a0a, _bh0_0, _bl0_0, 0)
359                          /*
360                           * 0 .. 18
361                           */
362                          VMAC(vqdmlsl.s32, _a0b, _bl0_1, _bl2_1, 0)
363                          /*
364                           * 2 .. 14
365                           */
366                          VMAC(vmlsl.s32, _a0b, _bl2_0, _bl2_0, 0)
367                          /*
368                           * 3 .. 14
369                           */
370                          VMAC(vmlal.s32, _a0b, _bl0_0, _bs0_0, 0)
371                          /*
372                           * 4 .. 15
373                           */
374                          VOP2(vmov, _a1a, _a1b)
375                          VMAC(vqdmlal.s32, _a1a, _bh0_1, _bh2_1, 1) /* 0 .. 18 */
376                          VMAC(vmlal.s32, _a1a, _bh2_0, _bh2_0, 1) /* 0 .. 19 */
377                          VMAC(vmlal.s32, _a1a, _bh0_0, _bl0_0, 1)
378                          /*
379                           * 0 .. 20
380                           */
381                          VOP2(vswp, _a0b_1, _a0a_0)
382
383                          VMAC(vqdmlsl.s32, _a1b, _bl0_1, _bl2_1, 1)
384                          /*
385                           * 2 .. 16
386                           */
387                          VMAC(vmlsl.s32, _a1b, _bl2_0, _bl2_0, 1)
388                          /*
389                           * 3 .. 16
390                           */
391                          VMAC(vmlal.s32, _a1b, _bl0_0, _bs0_0, 1)
392                          /*
393                           * 4 .. 17
394                           */
395                          VOP3(vsra.u64, _a0a, _a0b, "#28")
396                          VOP3(vsub.i32, _bs0_1, _bl0_1, _bh0_1)
397                          VOP2(vmovn.i64, _a0b_0, _a0b)
398
399                          VOP2(vswp, _a1b_1, _a1a_0)
400                          VOP3(vadd.i64, _a1b, _a0a, _a1b)
401
402                          VMAC(vqdmull.s32, _a0a, _as2_0, _bs2_1, 0) /* 0 .. 8 */
403                          VOP2(vmovn.i64, _a0b_1, _a1b)
404                          VOP3(vsra.u64, _a1a, _a1b, "#28")
405                          VMAC(vqdmlal.s32, _a0a, _as0_0, _bh0_1, 0) /* 0 .. 12 */
406                          VOP2(vbic.i32, _a0b, "#0xf0000000")
407                          "vstmia %[c]!, {" _a0b_0 ", " _a0b_1 "}" "\n\t" VMAC(vqdmull.s32, _a1b, _as2_0, _bs2_1, 1) /* 0
408                                                                                                                      * *
409                                                                                                                      * ..
410                                                                                                                      * * 8 */
411                          VMAC(vqdmlal.s32, _a1b, _as0_0, _bh0_1, 1)
412                          /*
413                           * 0 .. 12
414                           */
415                          VOP2(vmov, _a0b, _a0a) /* 0 .. 12 */
416                          VMAC(vqdmlal.s32, _a0a, _bh2_0, _bh2_1, 0) /* 0 .. 14 */
417                          VMAC(vqdmlal.s32, _a0a, _bh0_0, _bl0_1, 0)
418                          /*
419                           * 0 .. 16
420                           */
421                          VMAC(vqdmlsl.s32, _a0b, _bl2_0, _bl2_1, 0)
422                          /*
423                           * 2 .. 12
424                           */
425                          VMAC(vqdmlal.s32, _a0b, _bl0_0, _bs0_1, 0)
426                          /*
427                           * 4 .. 14
428                           */
429                          VOP3(vadd.i64, _a0a_0, _a0a_0, _a1a_1)
430                          VOP3(vadd.i64, _a0b_0, _a0b_0, _a1a_0)
431
432                          VOP2(vmov, _a1a, _a1b) /* 0 .. 12 */
433                          VMAC(vqdmlal.s32, _a1a, _bh2_0, _bh2_1, 1) /* 0 .. 14 */
434                          VMAC(vqdmlal.s32, _a1a, _bh0_0, _bl0_1, 1)
435                          /*
436                           * 0 .. 16
437                           */
438                          VOP2(vswp, _a0b_1, _a0a_0)
439
440                          VMAC(vqdmlsl.s32, _a1b, _bl2_0, _bl2_1, 1)
441                          /*
442                           * 2 .. 12
443                           */
444                          VMAC(vqdmlal.s32, _a1b, _bl0_0, _bs0_1, 1)
445                          /*
446                           * 4 .. 14
447                           */
448                          VOP3(vsra.u64, _a0a, _a0b, "#28")
449                          VOP3(vsub.i32, _bs2_0, _bl2_0, _bh2_0)
450                          VOP2(vmovn.i64, _a0b_0, _a0b)
451
452                          VOP2(vswp, _a1b_1, _a1a_0)
453                          VOP3(vadd.i64, _a1b, _a0a, _a1b)
454
455                          VMAC(vmull.s32, _a0a, _as2_1, _bs2_1, 0)
456                          VOP2(vmovn.i64, _a0b_1, _a1b)
457                          VMAC(vqdmlal.s32, _a0a, _as0_0, _bh2_0, 0)
458                          VOP3(vsra.u64, _a1a, _a1b, "#28")
459                          VMAC(vmlal.s32, _a0a, _as0_1, _bh0_1, 0)
460                          VOP2(vbic.i32, _a0b, "#0xf0000000")
461                          "vstmia %[c]!, {" _a0b_0 ", " _a0b_1 "}" "\n\t"
462                          VMAC(vmull.s32, _a1b, _as2_1, _bs2_1, 1)
463                          VMAC(vqdmlal.s32, _a1b, _as0_0, _bh2_0, 1)
464                          VMAC(vmlal.s32, _a1b, _as0_1, _bh0_1, 1)
465
466                          VOP2(vmov, _a0b_1, _a0a_1)
467                          VOP3(vadd.i64, _a0b_0, _a0a_0, _a1a_0)
468                          VOP3(vadd.i64, _a0a_0, _a0a_0, _a1a_1)
469                          VMAC(vmlal.s32, _a0a, _bh2_1, _bh2_1, 0)
470                          VMAC(vqdmlal.s32, _a0a, _bh0_0, _bl2_0, 0)
471                          VMAC(vmlal.s32, _a0a, _bh0_1, _bl0_1, 0)
472
473                          VMAC(vmlsl.s32, _a0b, _bl2_1, _bl2_1, 0)
474                          VMAC(vqdmlal.s32, _a0b, _bl0_0, _bs2_0, 0)
475                          VMAC(vmlal.s32, _a0b, _bl0_1, _bs0_1, 0)
476
477                          VOP2(vmov, _a1a, _a1b)
478                          VMAC(vmlal.s32, _a1a, _bh2_1, _bh2_1, 1)
479                          VMAC(vqdmlal.s32, _a1a, _bh0_0, _bl2_0, 1)
480                          VMAC(vmlal.s32, _a1a, _bh0_1, _bl0_1, 1)
481
482                          VOP2(vswp, _a0b_1, _a0a_0)
483
484                          VMAC(vmlsl.s32, _a1b, _bl2_1, _bl2_1, 1)
485                          VMAC(vqdmlal.s32, _a1b, _bl0_0, _bs2_0, 1)
486                          VMAC(vmlal.s32, _a1b, _bl0_1, _bs0_1, 1)
487
488                          VOP3(vsub.i32, _bs2_1, _bl2_1, _bh2_1)
489                          VOP3(vsra.u64, _a0a, _a0b, "#28")
490                          VOP2(vmovn.i64, _a0b_0, _a0b)
491
492                          VOP2(vswp, _a1b_1, _a1a_0)
493                          VOP3(vadd.i64, _a1b, _a0a, _a1b)
494
495                          VMAC(vqdmull.s32, _a0a, _as0_0, _bh2_1, 0)
496                          VOP2(vmovn.i64, _a0b_1, _a1b)
497                          VOP3(vsra.u64, _a1a, _a1b, "#28")
498                          VMAC(vqdmlal.s32, _a0a, _as2_0, _bh0_1, 0)
499                          VOP2(vbic.i32, _a0b, "#0xf0000000")
500                          "vstmia %[c]!, {" _a0b_0 ", " _a0b_1 "}" "\n\t"
501                          VMAC(vqdmull.s32, _a1b, _as0_0, _bh2_1, 1)
502                          VMAC(vqdmlal.s32, _a1b, _as2_0, _bh0_1, 1)
503
504                          VOP2(vmov, _a0b_1, _a0a_1)
505                          VOP3(vadd.i64, _a0b_0, _a0a_0, _a1a_0)
506                          VOP3(vadd.i64, _a0a_0, _a0a_0, _a1a_1)
507                          VMAC(vqdmlal.s32, _a0a, _bh0_0, _bl2_1, 0)
508                          VMAC(vqdmlal.s32, _a0a, _bh2_0, _bl0_1, 0)
509
510                          VMAC(vqdmlal.s32, _a0b, _bl0_0, _bs2_1, 0)
511                          VMAC(vqdmlal.s32, _a0b, _bl2_0, _bs0_1, 0)
512
513                          VOP2(vmov, _a1a, _a1b)
514                          VMAC(vqdmlal.s32, _a1a, _bh0_0, _bl2_1, 1)
515                          VMAC(vqdmlal.s32, _a1a, _bh2_0, _bl0_1, 1)
516
517                          VOP2(vswp, _a0b_1, _a0a_0)
518
519                          VMAC(vqdmlal.s32, _a1b, _bl0_0, _bs2_1, 1)
520                          VMAC(vqdmlal.s32, _a1b, _bl2_0, _bs0_1, 1)
521
522                          VOP3(vsra.u64, _a0a, _a0b, "#28")
523                          VOP2(vmovn.i64, _a0b_0, _a0b)
524
525                          VOP2(vswp, _a1b_1, _a1a_0)
526                          VOP3(vadd.i64, _a0a, _a0a, _a1b)
527
528                          VOP2(vmovn.i64, _a0b_1, _a0a)
529                          VOP3(vsra.u64, _a1a, _a0a, "#28")
530
531                          VOP2(vbic.i32, _a0b, "#0xf0000000")
532
533                          VOP2(vswp, _a1a_0, _a1a_1)
534
535                          "vstmia %[c]!, {" _a0b_0 ", " _a0b_1 "}" "\n\t"
536                          "sub %[c], #64" "\n\t"
537                          VOP3(vadd.i64, _a1a_1, _a1a_1, _a1a_0)
538
539                          "vldmia %[c], {" _a0a_0 ", " _a0a_1 ", " _a0b_0 "}"
540                          "\n\t" VOP2(vaddw.s32, _a1a, _a0a_0)
541                          VOP2(vmovn.i64, _a0a_0, _a1a)
542                          VOP2(vshr.s64, _a1a, "#28")
543
544                          VOP2(vaddw.s32, _a1a, _a0a_1)
545                          VOP2(vmovn.i64, _a0a_1, _a1a)
546                          VOP2(vshr.s64, _a1a, "#28")
547
548                          VOP2(vbic.i32, _a0a, "#0xf0000000")
549
550                          VOP2(vaddw.s32, _a1a, _a0b_0)
551                          VOP2(vmovn.i64, _a0b_0, _a1a)
552
553                          "vstmia %[c], {" _a0a_0 ", " _a0a_1 ", " _a0b_0 "}"
554                          "\n\t":[b] "+r"(bs)
555                          ,[c] "+r"(vc)
556
557                          ::"q0", "q1", "q2", "q3",
558                          "q4", "q5", "q6", "q7",
559                          "q12", "q13", "q14", "q15", "memory");
560 }
561
562 void gf_mulw_unsigned(gf_s * __restrict__ cs, const gf as, uint32_t b)
563 {
564     uint32x2_t vmask = { (1 << 28) - 1, (1 << 28) - 1 };
565     assert(b < (1 << 28));
566
567     uint64x2_t accum;
568     const uint32x2_t *va = (const uint32x2_t *)as->limb;
569     uint32x2_t *vo = (uint32x2_t *) cs->limb;
570     uint32x2_t vc, vn;
571     uint32x2_t vb = { b, 0 };
572
573     vc = va[0];
574     accum = vmull_lane_u32(vc, vb, 0);
575     vo[0] = vmovn_u64(accum) & vmask;
576     accum = vshrq_n_u64(accum, 28);
577
578     /*
579      * PERF: the right way to do this is to reduce behind, i.e. vmull + vmlal
580      * round 0 vmull + vmlal round 1 vmull + vmlal round 2 vsraq round 0, 1
581      * vmull + vmlal round 3 vsraq round 1, 2 ...
582      */
583
584     int i;
585     for (i = 1; i < 8; i++) {
586         vn = va[i];
587         accum = vmlal_lane_u32(accum, vn, vb, 0);
588         vo[i] = vmovn_u64(accum) & vmask;
589         accum = vshrq_n_u64(accum, 28);
590         vc = vn;
591     }
592
593     accum = xx_vaddup_u64(vrev128_u64(accum));
594     accum = vaddw_u32(accum, vo[0]);
595     vo[0] = vmovn_u64(accum) & vmask;
596
597     accum = vshrq_n_u64(accum, 28);
598     vo[1] += vmovn_u64(accum);
599 }