Add intel simd

Author: Raimo33

src/field_5x52_impl.h

@@ -14,6 +14,10 @@
 
 #include "field_5x52_int128_impl.h"
 
+#ifdef X86
+# include <immintrin.h>
+#endif
+
 #ifdef VERIFY
 static void secp256k1_fe_impl_verify(const secp256k1_fe *a) {
     const uint64_t *d = a->n;
@@ -37,10 +41,15 @@ static void secp256k1_fe_impl_get_bounds(secp256k1_fe *r, int m) {
     const uint64_t bound1 = 0xFFFFFFFFFFFFFULL * two_m;
     const uint64_t bound2 = 0x0FFFFFFFFFFFFULL * two_m;
 
+#ifdef __AVX__
+    __m256i vec = _mm256_set1_epi64x(bound1);
+    _mm256_storeu_si256((__m256i *)r->n, vec);
+#else
     r->n[0] = bound1;
     r->n[1] = bound1;
     r->n[2] = bound1;
     r->n[3] = bound1;
+#endif
     r->n[4] = bound2;
 }
 
@@ -239,6 +248,8 @@ static void secp256k1_fe_impl_set_b32_mod(secp256k1_fe *r, const unsigned char *
     limbs[3] = BYTESWAP_64(limbs[3]);
 #endif
 
+    /* TODO: parallelize avx2 */
+
     r->n[0] =                     (limbs[3] & 0xFFFFFFFFFFFFFULL);
     r->n[1] = (limbs[3] >> 52) | ((limbs[2] & 0xFFFFFFFFFFULL) << 12);
     r->n[2] = (limbs[2] >> 40) | ((limbs[1] & 0xFFFFFFFULL) << 24);
@@ -291,6 +302,10 @@ static void secp256k1_fe_impl_get_b32(unsigned char *r, const secp256k1_fe *a) {
 }
 
 SECP256K1_INLINE static void secp256k1_fe_impl_negate_unchecked(secp256k1_fe *r, const secp256k1_fe *a, int m) {
+#if defined(__AVX__) && defined(__AVX2__)
+    /* load here to mitigate load latency */
+    __m256i vec_a = _mm256_loadu_si256((__m256i *)a->n);
+#endif
     const uint32_t two_m1 = 2 * (m + 1);
     const uint64_t bound1 = 0xFFFFEFFFFFC2FULL * two_m1;
     const uint64_t bound2 = 0xFFFFFFFFFFFFFULL * two_m1;
@@ -303,10 +318,18 @@ SECP256K1_INLINE static void secp256k1_fe_impl_negate_unchecked(secp256k1_fe *r,
 
     /* Due to the properties above, the left hand in the subtractions below is never less than
      * the right hand. */
+#if defined(__AVX__) && defined(__AVX2__)
+    {
+        __m256i vec_bounds = _mm256_setr_epi64x(bound1, bound2, bound2, bound2);
+        __m256i out = _mm256_sub_epi64(vec_bounds, vec_a);
+        _mm256_storeu_si256((__m256i *)r->n, out);
+    }
+#else
     r->n[0] = bound1 - a->n[0];
     r->n[1] = bound2 - a->n[1];
     r->n[2] = bound2 - a->n[2];
     r->n[3] = bound2 - a->n[3];
+#endif
     r->n[4] = bound3 - a->n[4];
 }
 
@@ -339,15 +362,32 @@ SECP256K1_INLINE static void secp256k1_fe_impl_sqr(secp256k1_fe *r, const secp25
 }
 
 SECP256K1_INLINE static void secp256k1_fe_impl_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag) {
+#if defined(__AVX__) && defined(__AVX2__)
+    /* load here to mitigate load latency */
+    __m256i vec_r = _mm256_loadu_si256((__m256i *)(r->n));
+    __m256i vec_a = _mm256_loadu_si256((__m256i *)(a->n));
+#endif
+
     uint64_t mask0, mask1;
     volatile int vflag = flag;
     SECP256K1_CHECKMEM_CHECK_VERIFY(r->n, sizeof(r->n));
     mask0 = vflag + ~((uint64_t)0);
     mask1 = ~mask0;
+
+#if defined(__AVX__) && defined(__AVX2__)
+    {
+        __m256i vec_mask0 = _mm256_set1_epi64x(mask0);
+        __m256i vec_mask1 = _mm256_set1_epi64x(mask1);
+        vec_r = _mm256_and_si256(vec_r, vec_mask0);
+        vec_a = _mm256_and_si256(vec_a, vec_mask1);
+        _mm256_storeu_si256((__m256i *)r->n, _mm256_or_si256(vec_r, vec_a));
+    }
+#else
     r->n[0] = (r->n[0] & mask0) | (a->n[0] & mask1);
     r->n[1] = (r->n[1] & mask0) | (a->n[1] & mask1);
     r->n[2] = (r->n[2] & mask0) | (a->n[2] & mask1);
     r->n[3] = (r->n[3] & mask0) | (a->n[3] & mask1);
+#endif
     r->n[4] = (r->n[4] & mask0) | (a->n[4] & mask1);
 }
 
@@ -418,19 +458,42 @@ static SECP256K1_INLINE void secp256k1_fe_storage_cmov(secp256k1_fe_storage *r,
 }
 
 static void secp256k1_fe_impl_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a) {
+#if defined(__AVX__) && defined(__AVX2__)
+    __m256i limbs_0123 = _mm256_loadu_si256((__m256i *)a->n);
+    __m256i limbs_1234 = _mm256_loadu_si256((__m256i *)(a->n + 1));
+    const __m256i shift_lhs = _mm256_setr_epi64x(0, 12, 24, 36); /* TODO: precompute */
+    const __m256i shift_rhs = _mm256_setr_epi64x(52, 40, 28, 16); /* TODO: precompute */
+    __m256i rhs = _mm256_sllv_epi64(limbs_1234, shift_rhs);
+    __m256i lhs = _mm256_srlv_epi64(limbs_0123, shift_lhs);
+    _mm256_storeu_si256((__m256i *)r->n, _mm256_or_si256(lhs, rhs));
+#else
     r->n[0] = a->n[0]       | a->n[1] << 52;
     r->n[1] = a->n[1] >> 12 | a->n[2] << 40;
     r->n[2] = a->n[2] >> 24 | a->n[3] << 28;
     r->n[3] = a->n[3] >> 36 | a->n[4] << 16;
+#endif
 }
 
 static SECP256K1_INLINE void secp256k1_fe_impl_from_storage(secp256k1_fe *r, const secp256k1_fe_storage *a) {
     const uint64_t a0 = a->n[0], a1 = a->n[1], a2 = a->n[2], a3 = a->n[3];
 
+#if defined(__AVX__) && defined(__AVX2__)
+    {
+        __m256i limbs_0123 = _mm256_setr_epi64x(a0, a1, a2, a3);
+        __m256i limbs_0012 = _mm256_setr_epi64x(a0, a0, a1, a2);
+        const __m256i shift_lhs = _mm256_setr_epi64x(64, 52, 40, 28); /* TODO: precompute */
+        const __m256i shift_rhs = _mm256_setr_epi64x(0, 12, 24, 36); /* TODO: precompute */
+        const __m256i mask52 = _mm256_set1_epi64x(0xFFFFFFFFFFFFFULL); /* TODO: precompute */
+        __m256i rhs = _mm256_and_si256(_mm256_sllv_epi64(limbs_0123, shift_rhs), mask52);
+        __m256i lhs = _mm256_srlv_epi64(limbs_0012, shift_lhs);
+        _mm256_storeu_si256((__m256i*)r->n, _mm256_or_si256(lhs, rhs));
+    }
+#else
     r->n[0] = a0 & 0xFFFFFFFFFFFFFULL;
     r->n[1] = a0 >> 52 | ((a1 << 12) & 0xFFFFFFFFFFFFFULL);
     r->n[2] = a1 >> 40 | ((a2 << 24) & 0xFFFFFFFFFFFFFULL);
     r->n[3] = a2 >> 28 | ((a3 << 36) & 0xFFFFFFFFFFFFFULL);
+#endif
     r->n[4] = a3 >> 16;
 }
 
@@ -447,21 +510,49 @@ static void secp256k1_fe_from_signed62(secp256k1_fe *r, const secp256k1_modinv64
     VERIFY_CHECK(a3 >> 62 == 0);
     VERIFY_CHECK(a4 >> 8 == 0);
 
+#if defined(__AVX__) && defined(__AVX2__)
+    {
+        __m256i limbs_0123 = _mm256_setr_epi64x(a0, a1, a2, a3);
+        __m256i limbs_0012 = _mm256_setr_epi64x(a0, a0, a1, a2);
+        const __m256i shift_lhs = _mm256_setr_epi64x(64, 52, 42, 32); /*TODO: precompute */
+        const __m256i shift_rhs = _mm256_setr_epi64x(0, 10, 20, 30); /*TODO: precompute */
+        const __m256i mask52 = _mm256_set1_epi64x(M52); /*TODO: precompute */
+        __m256i rhs = _mm256_sllv_epi64(limbs_0123, shift_rhs);
+        __m256i lhs = _mm256_srlv_epi64(limbs_0012, shift_lhs);
+        __m256i out = _mm256_or_si256(lhs, rhs);
+        _mm256_storeu_si256((__m256i*)r->n, _mm256_and_si256(out, mask52));
+    }
+#else
     r->n[0] =  a0                   & M52;
     r->n[1] = (a0 >> 52 | a1 << 10) & M52;
     r->n[2] = (a1 >> 42 | a2 << 20) & M52;
     r->n[3] = (a2 >> 32 | a3 << 30) & M52;
+#endif
     r->n[4] = (a3 >> 22 | a4 << 40);
 }
 
 static void secp256k1_fe_to_signed62(secp256k1_modinv64_signed62 *r, const secp256k1_fe *a) {
     const uint64_t M62 = UINT64_MAX >> 2;
     const uint64_t a0 = a->n[0], a1 = a->n[1], a2 = a->n[2], a3 = a->n[3], a4 = a->n[4];
 
+#if defined(__AVX__) && defined(__AVX2__)
+    {
+        __m256i limbs_0123 = _mm256_setr_epi64x(a0, a1, a2, a3);
+        __m256i limbs_1234 = _mm256_setr_epi64x(a1, a2, a3, a4);
+        const __m256i shift_lhs = _mm256_setr_epi64x(0, 10, 20, 30); /*TODO: precompute */
+        const __m256i shift_rhs = _mm256_setr_epi64x(52, 42, 32, 22); /*TODO: precompute */
+        const __m256i mask62 = _mm256_set1_epi64x(M62); /*TODO: precompute */
+        __m256i lhs = _mm256_srlv_epi64(limbs_0123, shift_lhs);
+        __m256i rhs = _mm256_sllv_epi64(limbs_1234, shift_rhs);
+        __m256i out = _mm256_or_si256(lhs, rhs);
+        _mm256_storeu_si256((__m256i *)r->v, _mm256_and_si256(out, mask62));
+    }
+#else
     r->v[0] = (a0       | a1 << 52) & M62;
     r->v[1] = (a1 >> 10 | a2 << 42) & M62;
     r->v[2] = (a2 >> 20 | a3 << 32) & M62;
     r->v[3] = (a3 >> 30 | a4 << 22) & M62;
+#endif
     r->v[4] =  a4 >> 40;
 }
 

src/hash_impl.h

@@ -14,6 +14,10 @@
 #include <stdint.h>
 #include <string.h>
 
+#ifdef X86
+# include <immintrin.h>
+#endif
+
 #define Ch(x,y,z) ((z) ^ ((x) & ((y) ^ (z))))
 #define Maj(x,y,z) (((x) & (y)) | ((z) & ((x) | (y))))
 #define Sigma0(x) (((x) >> 2  | (x) << 30) ^ ((x) >> 13 | (x) << 19) ^ ((x) >> 22 | (x) << 10))