Barretenberg
The ZK-SNARK library at the core of Aztec
Loading...
Searching...
No Matches
field_impl_x64.hpp
Go to the documentation of this file.
1// === AUDIT STATUS ===
2// internal: { status: not started, auditors: [], date: YYYY-MM-DD }
3// external_1: { status: not started, auditors: [], date: YYYY-MM-DD }
4// external_2: { status: not started, auditors: [], date: YYYY-MM-DD }
5// =====================
6
7#pragma once
8
9#if (BBERG_NO_ASM == 0)
10#include "./field_impl.hpp"
11#include "asm_macros.hpp"
12namespace bb {
13
14template <class T> field<T> field<T>::asm_mul_with_coarse_reduction(const field& a, const field& b) noexcept
15{
16 field r;
17 constexpr uint64_t r_inv = T::r_inv;
18 constexpr uint64_t modulus_0 = modulus.data[0];
19 constexpr uint64_t modulus_1 = modulus.data[1];
20 constexpr uint64_t modulus_2 = modulus.data[2];
21 constexpr uint64_t modulus_3 = modulus.data[3];
22 constexpr uint64_t zero_ref = 0;
23
33 __asm__(MUL("0(%0)", "8(%0)", "16(%0)", "24(%0)", "%1")
34 STORE_FIELD_ELEMENT("%2", "%%r12", "%%r13", "%%r14", "%%r15")
35 :
36 : "%r"(&a),
37 "%r"(&b),
38 "r"(&r),
39 [modulus_0] "m"(modulus_0),
40 [modulus_1] "m"(modulus_1),
41 [modulus_2] "m"(modulus_2),
42 [modulus_3] "m"(modulus_3),
43 [r_inv] "m"(r_inv),
44 [zero_reference] "m"(zero_ref)
45 : "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
46 return r;
47}
48
49template <class T> void field<T>::asm_self_mul_with_coarse_reduction(const field& a, const field& b) noexcept
50{
51 constexpr uint64_t r_inv = T::r_inv;
52 constexpr uint64_t modulus_0 = modulus.data[0];
53 constexpr uint64_t modulus_1 = modulus.data[1];
54 constexpr uint64_t modulus_2 = modulus.data[2];
55 constexpr uint64_t modulus_3 = modulus.data[3];
56 constexpr uint64_t zero_ref = 0;
66 __asm__(MUL("0(%0)", "8(%0)", "16(%0)", "24(%0)", "%1")
67 STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
68 :
69 : "r"(&a),
70 "r"(&b),
71 [modulus_0] "m"(modulus_0),
72 [modulus_1] "m"(modulus_1),
73 [modulus_2] "m"(modulus_2),
74 [modulus_3] "m"(modulus_3),
75 [r_inv] "m"(r_inv),
76 [zero_reference] "m"(zero_ref)
77 : "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
78}
79
80template <class T> field<T> field<T>::asm_sqr_with_coarse_reduction(const field& a) noexcept
81{
82 field r;
83 constexpr uint64_t r_inv = T::r_inv;
84 constexpr uint64_t modulus_0 = modulus.data[0];
85 constexpr uint64_t modulus_1 = modulus.data[1];
86 constexpr uint64_t modulus_2 = modulus.data[2];
87 constexpr uint64_t modulus_3 = modulus.data[3];
88 constexpr uint64_t zero_ref = 0;
89
90// Our SQR implementation with BMI2 but without ADX has a bug.
91// The case is extremely rare so fixing it is a bit of a waste of time.
92// We'll use MUL instead.
93#if !defined(__ADX__) || defined(DISABLE_ADX)
103 __asm__(MUL("0(%0)", "8(%0)", "16(%0)", "24(%0)", "%1")
104 STORE_FIELD_ELEMENT("%2", "%%r12", "%%r13", "%%r14", "%%r15")
105 :
106 : "%r"(&a),
107 "%r"(&a),
108 "r"(&r),
109 [modulus_0] "m"(modulus_0),
110 [modulus_1] "m"(modulus_1),
111 [modulus_2] "m"(modulus_2),
112 [modulus_3] "m"(modulus_3),
113 [r_inv] "m"(r_inv),
114 [zero_reference] "m"(zero_ref)
115 : "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
116
117#else
118
127 __asm__(SQR("%0")
128 // "movq %[r_ptr], %%rsi \n\t"
129 STORE_FIELD_ELEMENT("%1", "%%r12", "%%r13", "%%r14", "%%r15")
130 :
131 : "r"(&a),
132 "r"(&r),
133 [zero_reference] "m"(zero_ref),
134 [modulus_0] "m"(modulus_0),
135 [modulus_1] "m"(modulus_1),
136 [modulus_2] "m"(modulus_2),
137 [modulus_3] "m"(modulus_3),
138 [r_inv] "m"(r_inv)
139 : "%rcx", "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
140#endif
141 return r;
142}
143
144template <class T> void field<T>::asm_self_sqr_with_coarse_reduction(const field& a) noexcept
145{
146 constexpr uint64_t r_inv = T::r_inv;
147 constexpr uint64_t modulus_0 = modulus.data[0];
148 constexpr uint64_t modulus_1 = modulus.data[1];
149 constexpr uint64_t modulus_2 = modulus.data[2];
150 constexpr uint64_t modulus_3 = modulus.data[3];
151 constexpr uint64_t zero_ref = 0;
152
153// Our SQR implementation with BMI2 but without ADX has a bug.
154// The case is extremely rare so fixing it is a bit of a waste of time.
155// We'll use MUL instead.
156#if !defined(__ADX__) || defined(DISABLE_ADX)
166 __asm__(MUL("0(%0)", "8(%0)", "16(%0)", "24(%0)", "%1")
167 STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
168 :
169 : "r"(&a),
170 "r"(&a),
171 [modulus_0] "m"(modulus_0),
172 [modulus_1] "m"(modulus_1),
173 [modulus_2] "m"(modulus_2),
174 [modulus_3] "m"(modulus_3),
175 [r_inv] "m"(r_inv),
176 [zero_reference] "m"(zero_ref)
177 : "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
178
179#else
188 __asm__(SQR("%0")
189 // "movq %[r_ptr], %%rsi \n\t"
190 STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
191 :
192 : "r"(&a),
193 [zero_reference] "m"(zero_ref),
194 [modulus_0] "m"(modulus_0),
195 [modulus_1] "m"(modulus_1),
196 [modulus_2] "m"(modulus_2),
197 [modulus_3] "m"(modulus_3),
198 [r_inv] "m"(r_inv)
199 : "%rcx", "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
200#endif
201}
202
203template <class T> field<T> field<T>::asm_add_with_coarse_reduction(const field& a, const field& b) noexcept
204{
205 field r;
206
207 constexpr uint64_t twice_not_modulus_0 = twice_not_modulus.data[0];
208 constexpr uint64_t twice_not_modulus_1 = twice_not_modulus.data[1];
209 constexpr uint64_t twice_not_modulus_2 = twice_not_modulus.data[2];
210 constexpr uint64_t twice_not_modulus_3 = twice_not_modulus.data[3];
211
212 __asm__(CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
213 ADD_REDUCE("%1",
214 "%[twice_not_modulus_0]",
215 "%[twice_not_modulus_1]",
216 "%[twice_not_modulus_2]",
217 "%[twice_not_modulus_3]") STORE_FIELD_ELEMENT("%2", "%%r12", "%%r13", "%%r14", "%%r15")
218 :
219 : "%r"(&a),
220 "%r"(&b),
221 "r"(&r),
222 [twice_not_modulus_0] "m"(twice_not_modulus_0),
223 [twice_not_modulus_1] "m"(twice_not_modulus_1),
224 [twice_not_modulus_2] "m"(twice_not_modulus_2),
225 [twice_not_modulus_3] "m"(twice_not_modulus_3)
226 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
227 return r;
228}
229
230template <class T> void field<T>::asm_self_add_with_coarse_reduction(const field& a, const field& b) noexcept
231{
232 constexpr uint64_t twice_not_modulus_0 = twice_not_modulus.data[0];
233 constexpr uint64_t twice_not_modulus_1 = twice_not_modulus.data[1];
234 constexpr uint64_t twice_not_modulus_2 = twice_not_modulus.data[2];
235 constexpr uint64_t twice_not_modulus_3 = twice_not_modulus.data[3];
236
237 __asm__(CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
238 ADD_REDUCE("%1",
239 "%[twice_not_modulus_0]",
240 "%[twice_not_modulus_1]",
241 "%[twice_not_modulus_2]",
242 "%[twice_not_modulus_3]") STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
243 :
244 : "r"(&a),
245 "r"(&b),
246 [twice_not_modulus_0] "m"(twice_not_modulus_0),
247 [twice_not_modulus_1] "m"(twice_not_modulus_1),
248 [twice_not_modulus_2] "m"(twice_not_modulus_2),
249 [twice_not_modulus_3] "m"(twice_not_modulus_3)
250 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
251}
252
253template <class T> field<T> field<T>::asm_sub_with_coarse_reduction(const field& a, const field& b) noexcept
254{
255 field r;
256
257 constexpr uint64_t twice_modulus_0 = twice_modulus.data[0];
258 constexpr uint64_t twice_modulus_1 = twice_modulus.data[1];
259 constexpr uint64_t twice_modulus_2 = twice_modulus.data[2];
260 constexpr uint64_t twice_modulus_3 = twice_modulus.data[3];
261
262 __asm__(
263 CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15") SUB("%1")
264 REDUCE_FIELD_ELEMENT("%[twice_modulus_0]", "%[twice_modulus_1]", "%[twice_modulus_2]", "%[twice_modulus_3]")
265 STORE_FIELD_ELEMENT("%2", "%%r12", "%%r13", "%%r14", "%%r15")
266 :
267 : "r"(&a),
268 "r"(&b),
269 "r"(&r),
270 [twice_modulus_0] "m"(twice_modulus_0),
271 [twice_modulus_1] "m"(twice_modulus_1),
272 [twice_modulus_2] "m"(twice_modulus_2),
273 [twice_modulus_3] "m"(twice_modulus_3)
274 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
275 return r;
276}
277
278template <class T> void field<T>::asm_self_sub_with_coarse_reduction(const field& a, const field& b) noexcept
279{
280 constexpr uint64_t twice_modulus_0 = twice_modulus.data[0];
281 constexpr uint64_t twice_modulus_1 = twice_modulus.data[1];
282 constexpr uint64_t twice_modulus_2 = twice_modulus.data[2];
283 constexpr uint64_t twice_modulus_3 = twice_modulus.data[3];
284
285 __asm__(
286 CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15") SUB("%1")
287 REDUCE_FIELD_ELEMENT("%[twice_modulus_0]", "%[twice_modulus_1]", "%[twice_modulus_2]", "%[twice_modulus_3]")
288 STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
289 :
290 : "r"(&a),
291 "r"(&b),
292 [twice_modulus_0] "m"(twice_modulus_0),
293 [twice_modulus_1] "m"(twice_modulus_1),
294 [twice_modulus_2] "m"(twice_modulus_2),
295 [twice_modulus_3] "m"(twice_modulus_3)
296 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
297}
298
299template <class T> void field<T>::asm_conditional_negate(field& r, const uint64_t predicate) noexcept
300{
301 constexpr uint64_t twice_modulus_0 = twice_modulus.data[0];
302 constexpr uint64_t twice_modulus_1 = twice_modulus.data[1];
303 constexpr uint64_t twice_modulus_2 = twice_modulus.data[2];
304 constexpr uint64_t twice_modulus_3 = twice_modulus.data[3];
305
306 __asm__(CLEAR_FLAGS("%%r8") LOAD_FIELD_ELEMENT(
307 "%1", "%%r8", "%%r9", "%%r10", "%%r11") "movq %[twice_modulus_0], %%r12 \n\t"
308 "movq %[twice_modulus_1], %%r13 \n\t"
309 "movq %[twice_modulus_2], %%r14 \n\t"
310 "movq %[twice_modulus_3], %%r15 \n\t"
311 "subq %%r8, %%r12 \n\t"
312 "sbbq %%r9, %%r13 \n\t"
313 "sbbq %%r10, %%r14 \n\t"
314 "sbbq %%r11, %%r15 \n\t"
315 "testq %0, %0 \n\t"
316 "cmovnzq %%r12, %%r8 \n\t"
317 "cmovnzq %%r13, %%r9 \n\t"
318 "cmovnzq %%r14, %%r10 \n\t"
319 "cmovnzq %%r15, %%r11 \n\t" STORE_FIELD_ELEMENT(
320 "%1", "%%r8", "%%r9", "%%r10", "%%r11")
321 :
322 : "r"(predicate),
323 "r"(&r),
324 [twice_modulus_0] "i"(twice_modulus_0),
325 [twice_modulus_1] "i"(twice_modulus_1),
326 [twice_modulus_2] "i"(twice_modulus_2),
327 [twice_modulus_3] "i"(twice_modulus_3)
328 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
329}
330
331template <class T> field<T> field<T>::asm_reduce_once(const field& a) noexcept
332{
333 field r;
334
335 constexpr uint64_t not_modulus_0 = not_modulus.data[0];
336 constexpr uint64_t not_modulus_1 = not_modulus.data[1];
337 constexpr uint64_t not_modulus_2 = not_modulus.data[2];
338 constexpr uint64_t not_modulus_3 = not_modulus.data[3];
339
340 __asm__(CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
341 REDUCE_FIELD_ELEMENT("%[not_modulus_0]", "%[not_modulus_1]", "%[not_modulus_2]", "%[not_modulus_3]")
342 STORE_FIELD_ELEMENT("%1", "%%r12", "%%r13", "%%r14", "%%r15")
343 :
344 : "r"(&a),
345 "r"(&r),
346 [not_modulus_0] "m"(not_modulus_0),
347 [not_modulus_1] "m"(not_modulus_1),
348 [not_modulus_2] "m"(not_modulus_2),
349 [not_modulus_3] "m"(not_modulus_3)
350 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
351 return r;
352}
353
354template <class T> void field<T>::asm_self_reduce_once(const field& a) noexcept
355{
356 constexpr uint64_t not_modulus_0 = not_modulus.data[0];
357 constexpr uint64_t not_modulus_1 = not_modulus.data[1];
358 constexpr uint64_t not_modulus_2 = not_modulus.data[2];
359 constexpr uint64_t not_modulus_3 = not_modulus.data[3];
360
361 __asm__(CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
362 REDUCE_FIELD_ELEMENT("%[not_modulus_0]", "%[not_modulus_1]", "%[not_modulus_2]", "%[not_modulus_3]")
363 STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
364 :
365 : "r"(&a),
366 [not_modulus_0] "m"(not_modulus_0),
367 [not_modulus_1] "m"(not_modulus_1),
368 [not_modulus_2] "m"(not_modulus_2),
369 [not_modulus_3] "m"(not_modulus_3)
370 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
371}
372} // namespace bb
373#endif
asm_macros.hpp
ADD_REDUCE
#define ADD_REDUCE(b, modulus_0, modulus_1, modulus_2, modulus_3)
Definition asm_macros.hpp:63
STORE_FIELD_ELEMENT
#define STORE_FIELD_ELEMENT(r, lolo, lohi, hilo, hihi)
Definition asm_macros.hpp:31
LOAD_FIELD_ELEMENT
#define LOAD_FIELD_ELEMENT(a, lolo, lohi, hilo, hihi)
Definition asm_macros.hpp:20
CLEAR_FLAGS
#define CLEAR_FLAGS(empty_reg)
Definition asm_macros.hpp:13
REDUCE_FIELD_ELEMENT
#define REDUCE_FIELD_ELEMENT(neg_modulus_0, neg_modulus_1, neg_modulus_2, neg_modulus_3)
Definition asm_macros.hpp:87
a
FF a
Definition field_gt.test.cpp:51
b
FF b
Definition field_gt.test.cpp:52
field_impl.hpp
bb
Entry point for Barretenberg command-line interface.
Definition acir_format_getters.cpp:6
bb::Instruction::SQR
@ SQR
Square a field element.
bb::Instruction::SUB
@ SUB
Subtract two field elements.
bb::Instruction::MUL
@ MUL
Multiply two field elements.