Move dynamic dispatching to C for x86_64-mont5.pl

crypto/fipsmodule/bn/asm/x86_64-mont5.pl

@@ -54,7 +54,7 @@
 $addx = 1;
 for (@ARGV) { $addx = 0 if (/-DMY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX/); }
 
-# int bn_mul_mont_gather5(
+# int bn_mul_mont_gather5_nohw(
 $rp="%rdi";	# BN_ULONG *rp,
 $ap="%rsi";	# const BN_ULONG *ap,
 $bp="%rdx";	# const BN_ULONG *bp,
@@ -76,31 +76,15 @@
 $code=<<___;
 .text
 
-.extern	OPENSSL_ia32cap_P
-
-.globl	bn_mul_mont_gather5
-.type	bn_mul_mont_gather5,\@function,6
+.globl	bn_mul_mont_gather5_nohw
+.type	bn_mul_mont_gather5_nohw,\@function,6
 .align	64
-bn_mul_mont_gather5:
+bn_mul_mont_gather5_nohw:
 .cfi_startproc
 	_CET_ENDBR
 	mov	${num}d,${num}d
 	mov	%rsp,%rax
 .cfi_def_cfa_register	%rax
-	test	\$7,${num}d
-	jnz	.Lmul_enter
-___
-$code.=<<___ if ($addx);
-#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
-	leaq	OPENSSL_ia32cap_P(%rip),%r11
-	mov	8(%r11),%r11d
-#endif
-___
-$code.=<<___;
-	jmp	.Lmul4x_enter
-
-.align	16
-.Lmul_enter:
 	movd	`($win64?56:8)`(%rsp),%xmm5	# load 7th argument
 	push	%rbx
 .cfi_push	%rbx
@@ -460,29 +444,21 @@
 .Lmul_epilogue:
 	ret
 .cfi_endproc
-.size	bn_mul_mont_gather5,.-bn_mul_mont_gather5
+.size	bn_mul_mont_gather5_nohw,.-bn_mul_mont_gather5_nohw
 ___
 {{{
 my @A=("%r10","%r11");
 my @N=("%r13","%rdi");
 $code.=<<___;
+.global bn_mul4x_mont_gather5
 .type	bn_mul4x_mont_gather5,\@function,6
 .align	32
 bn_mul4x_mont_gather5:
 .cfi_startproc
+    _CET_ENDBR
 	.byte	0x67
 	mov	%rsp,%rax
 .cfi_def_cfa_register	%rax
-.Lmul4x_enter:
-___
-$code.=<<___ if ($addx);
-#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
-	and	\$0x80108,%r11d
-	cmp	\$0x80108,%r11d		# check for AD*X+BMI2+BMI1
-	je	.Lmulx4x_enter
-#endif
-___
-$code.=<<___;
 	push	%rbx
 .cfi_push	%rbx
 	push	%rbp
@@ -1087,7 +1063,7 @@
 }}}
 {{{
 ######################################################################
-# void bn_power5(
+# void bn_power5_nohw(
 my $rptr="%rdi";	# BN_ULONG *rptr,
 my $aptr="%rsi";	# const BN_ULONG *aptr,
 my $bptr="%rdx";	# const BN_ULONG *table,
@@ -1102,25 +1078,14 @@
 my ($a0,$a1,$ai)=("%r14","%r15","%rbx");
 
 $code.=<<___;
-.globl	bn_power5
-.type	bn_power5,\@function,6
+.globl	bn_power5_nohw
+.type	bn_power5_nohw,\@function,6
 .align	32
-bn_power5:
+bn_power5_nohw:
 .cfi_startproc
 	_CET_ENDBR
 	mov	%rsp,%rax
 .cfi_def_cfa_register	%rax
-___
-$code.=<<___ if ($addx);
-#ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
-	leaq	OPENSSL_ia32cap_P(%rip),%r11
-	mov	8(%r11),%r11d
-	and	\$0x80108,%r11d
-	cmp	\$0x80108,%r11d		# check for AD*X+BMI2+BMI1
-	je	.Lpowerx5_enter
-#endif
-___
-$code.=<<___;
 	push	%rbx
 .cfi_push	%rbx
 	push	%rbp
@@ -1243,7 +1208,7 @@
 .Lpower5_epilogue:
 	ret
 .cfi_endproc
-.size	bn_power5,.-bn_power5
+.size	bn_power5_nohw,.-bn_power5_nohw
 
 .globl	bn_sqr8x_internal
 .hidden	bn_sqr8x_internal
@@ -2119,13 +2084,14 @@
 
 $code.=<<___;
 #ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
+.global bn_mulx4x_mont_gather5
 .type	bn_mulx4x_mont_gather5,\@function,6
 .align	32
 bn_mulx4x_mont_gather5:
 .cfi_startproc
+    _CET_ENDBR
 	mov	%rsp,%rax
 .cfi_def_cfa_register	%rax
-.Lmulx4x_enter:
 	push	%rbx
 .cfi_push	%rbx
 	push	%rbp
@@ -2594,7 +2560,7 @@
 ___
 }{
 ######################################################################
-# void bn_power5(
+# void bn_powerx5(
 my $rptr="%rdi";	# BN_ULONG *rptr,
 my $aptr="%rsi";	# const BN_ULONG *aptr,
 my $bptr="%rdx";	# const BN_ULONG *table,
@@ -2609,13 +2575,14 @@
 my ($a0,$a1,$ai)=("%r14","%r15","%rbx");
 
 $code.=<<___;
+.global bn_powerx5
 .type	bn_powerx5,\@function,6
 .align	32
 bn_powerx5:
 .cfi_startproc
+    _CET_ENDBR
 	mov	%rsp,%rax
 .cfi_def_cfa_register	%rax
-.Lpowerx5_enter:
 	push	%rbx
 .cfi_push	%rbx
 	push	%rbp
@@ -3717,17 +3684,17 @@
 
 .section	.pdata
 .align	4
-	.rva	.LSEH_begin_bn_mul_mont_gather5
-	.rva	.LSEH_end_bn_mul_mont_gather5
-	.rva	.LSEH_info_bn_mul_mont_gather5
+	.rva	.LSEH_begin_bn_mul_mont_gather5_nohw
+	.rva	.LSEH_end_bn_mul_mont_gather5_nohw
+	.rva	.LSEH_info_bn_mul_mont_gather5_nohw
 
 	.rva	.LSEH_begin_bn_mul4x_mont_gather5
 	.rva	.LSEH_end_bn_mul4x_mont_gather5
 	.rva	.LSEH_info_bn_mul4x_mont_gather5
 
-	.rva	.LSEH_begin_bn_power5
-	.rva	.LSEH_end_bn_power5
-	.rva	.LSEH_info_bn_power5
+	.rva	.LSEH_begin_bn_power5_nohw
+	.rva	.LSEH_end_bn_power5_nohw
+	.rva	.LSEH_info_bn_power5_nohw
 ___
 $code.=<<___ if ($addx);
 #ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
@@ -3747,7 +3714,7 @@
 
 .section	.xdata
 .align	4
-.LSEH_info_bn_mul_mont_gather5:
+.LSEH_info_bn_mul_mont_gather5_nohw:
 	.byte	9,0,0,0
 	.rva	mul_handler
 	.rva	.Lmul_body,.Lmul_body,.Lmul_epilogue		# HandlerData[]
@@ -3757,7 +3724,7 @@
 	.rva	mul_handler
 	.rva	.Lmul4x_prologue,.Lmul4x_body,.Lmul4x_epilogue		# HandlerData[]
 .align	4
-.LSEH_info_bn_power5:
+.LSEH_info_bn_power5_nohw:
 	.byte	9,0,0,0
 	.rva	mul_handler
 	.rva	.Lpower5_prologue,.Lpower5_body,.Lpower5_epilogue	# HandlerData[]

crypto/fipsmodule/bn/bn_test.cc

@@ -2959,14 +2959,12 @@ TEST_F(BNTest, BNMulMontABI) {
     b[0] = 42;
 
 #if defined(OPENSSL_X86_64)
-#if !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
     if (bn_mulx4x_mont_capable(words)) {
       CHECK_ABI(bn_mulx4x_mont, r.data(), a.data(), b.data(), mont->N.d,
                 mont->n0, words);
       CHECK_ABI(bn_mulx4x_mont, r.data(), a.data(), a.data(), mont->N.d,
                 mont->n0, words);
     }
-#endif // !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
     if (bn_mul4x_mont_capable(words)) {
       CHECK_ABI(bn_mul4x_mont, r.data(), a.data(), b.data(), mont->N.d,
                 mont->n0, words);
@@ -2977,12 +2975,10 @@ TEST_F(BNTest, BNMulMontABI) {
               mont->n0, words);
     CHECK_ABI(bn_mul_mont_nohw, r.data(), a.data(), a.data(), mont->N.d,
               mont->n0, words);
-#if !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
     if (bn_sqr8x_mont_capable(words)) {
       CHECK_ABI(bn_sqr8x_mont, r.data(), a.data(), bn_mulx_adx_capable(),
                 mont->N.d, mont->n0, words);
     }
-#endif // !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
 #elif defined(OPENSSL_ARM)
     if (bn_mul8x_mont_neon_capable(words)) {
       CHECK_ABI(bn_mul8x_mont_neon, r.data(), a.data(), b.data(), mont->N.d,
@@ -3028,18 +3024,35 @@ TEST_F(BNTest, BNMulMont5ABI) {
       bn_scatter5(r.data(), words, table.data(), i);
     }
     CHECK_ABI(bn_gather5, r.data(), words, table.data(), 13);
-
-    CHECK_ABI(bn_mul_mont_gather5, r.data(), r.data(), table.data(), m->d,
+    if (bn_mulx4x_mont_gather5_capable(words)) {
+      CHECK_ABI(bn_mulx4x_mont_gather5, r.data(), r.data(), table.data(), m->d,
+                mont->n0, words, 13);
+      CHECK_ABI(bn_mulx4x_mont_gather5, r.data(), a.data(), table.data(), m->d,
+                mont->n0, words, 13);
+    }
+    if (bn_mul4x_mont_gather5_capable(words)) {
+      CHECK_ABI(bn_mul4x_mont_gather5, r.data(), r.data(), table.data(), m->d,
+                mont->n0, words, 13);
+      CHECK_ABI(bn_mul4x_mont_gather5, r.data(), a.data(), table.data(), m->d,
+                mont->n0, words, 13);
+    }
+    CHECK_ABI(bn_mul_mont_gather5_nohw, r.data(), r.data(), table.data(), m->d,
               mont->n0, words, 13);
-    CHECK_ABI(bn_mul_mont_gather5, r.data(), a.data(), table.data(), m->d,
+    CHECK_ABI(bn_mul_mont_gather5_nohw, r.data(), a.data(), table.data(), m->d,
               mont->n0, words, 13);
 
-    if (words % 8 == 0) {
-      CHECK_ABI(bn_power5, r.data(), r.data(), table.data(), m->d, mont->n0,
+    if (bn_powerx5_capable(words)) {
+      CHECK_ABI(bn_powerx5, r.data(), r.data(), table.data(), m->d, mont->n0,
                 words, 13);
-      CHECK_ABI(bn_power5, r.data(), a.data(), table.data(), m->d, mont->n0,
+      CHECK_ABI(bn_powerx5, r.data(), a.data(), table.data(), m->d, mont->n0,
                 words, 13);
     }
+    if (bn_power5_capable(words)) {
+      CHECK_ABI(bn_power5_nohw, r.data(), r.data(), table.data(), m->d,
+                mont->n0, words, 13);
+      CHECK_ABI(bn_power5_nohw, r.data(), a.data(), table.data(), m->d,
+                mont->n0, words, 13);
+    }
   }
 }
 #endif  // OPENSSL_BN_ASM_MONT5 && SUPPORTS_ABI_TEST

crypto/fipsmodule/bn/exponentiation.c

@@ -107,6 +107,7 @@
  * Hudson ([email protected]). */
 
 #include <openssl/bn.h>
+#include <openssl/cpu.h>
 
 #include <assert.h>
 #include <limits.h>
@@ -163,6 +164,56 @@ static void exponentiation_s2n_bignum_copy_from_prebuf(BN_ULONG *dest, int width
 #endif
 }
 
+#if defined(OPENSSL_BN_ASM_MONT5)
+
+// bn_mul_mont_gather5 multiples loads index |power| of |table|, multiplies it
+// by |ap| modulo |np|, and stores the result in |rp|. The values are |num|
+// words long and represented in Montgomery form. |n0| is a pointer to the
+// corresponding field in |BN_MONT_CTX|. |table| must be aligned to at least
+// 16 bytes. |power| must be less than 32 and is treated as secret.
+//
+// WARNING: This function implements Almost Montgomery Multiplication from
+// https://eprint.iacr.org/2011/239. The inputs do not need to be fully reduced.
+// However, even if they are fully reduced, the output may not be.
+static void bn_mul_mont_gather5(
+    BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *table, const BN_ULONG *np,
+    const BN_ULONG *n0, int num, int power) {
+  if (bn_mulx4x_mont_gather5_capable(num)) {
+    log_dispatch(15);
+    bn_mulx4x_mont_gather5(rp, ap, table, np, n0, num, power);
+  } else if (bn_mul4x_mont_gather5_capable(num)) {
+    log_dispatch(16);
+    bn_mul4x_mont_gather5(rp, ap, table, np, n0, num, power);
+  } else {
+    log_dispatch(17);
+    bn_mul_mont_gather5_nohw(rp, ap, table, np, n0, num, power);
+  }
+}
+
+// bn_power5 squares |ap| five times and multiplies it by the value stored at
+// index |power| of |table|, modulo |np|. It stores the result in |rp|. The
+// values are |num| words long and represented in Montgomery form. |n0| is a
+// pointer to the corresponding field in |BN_MONT_CTX|. |num| must be divisible
+// by 8. |power| must be less than 32 and is treated as secret.
+//
+// WARNING: This function implements Almost Montgomery Multiplication from
+// https://eprint.iacr.org/2011/239. The inputs do not need to be fully reduced.
+// However, even if they are fully reduced, the output may not be.
+static void bn_power5(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *table,
+                      const BN_ULONG *np, const BN_ULONG *n0, int num,
+                      int power)
+{
+  assert(bn_power5_capable(num));
+  if (bn_powerx5_capable(num)) {
+    log_dispatch(18);
+    bn_powerx5(rp, ap, table, np, n0, num, power);
+  } else {
+    log_dispatch(19);
+    bn_power5_nohw(rp, ap, table, np, n0, num, power);
+  }
+}
+
+#endif // defined(OPENSSL_BN_ASM_MONT5)
 
 int BN_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) {
   int i, bits, ret = 0;
@@ -1122,7 +1173,7 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
 
     // Scan the exponent one window at a time starting from the most
     // significant bits.
-    if (top & 7) {
+    if (!bn_power5_capable(top)) {
       while (bits >= 0) {
         for (wvalue = 0, i = 0; i < 5; i++, bits--) {
           wvalue = (wvalue << 1) + BN_is_bit_set(p, bits);

crypto/fipsmodule/bn/generic.c

@@ -622,3 +622,38 @@ BN_ULONG bn_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b,
 }
 
 #endif  // !BN_ADD_ASM
+
+// Default implementations of hardware accelerated functions that throw errors if reached.
+// Used to reduce the number of guards needed in code.
+#if defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)
+
+#if defined(OPENSSL_X86_64) && defined(OPENSSL_BN_ASM_MONT)
+int bn_mulx4x_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
+                   const BN_ULONG *np, const BN_ULONG *n0, size_t num) {
+  perror("bn_mulx4x_mont");
+  abort();
+}
+#endif
+
+#if defined(OPENSSL_BN_ASM_MONT5)
+void bn_mulx4x_mont_gather5(BN_ULONG *rp, const BN_ULONG *ap,
+                             const BN_ULONG *table, const BN_ULONG *np,
+                             const BN_ULONG *n0, int num, int power) {
+  perror("bn_mulx4x_mont_gather5");
+  abort();
+}
+
+void bn_powerx5(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *table,
+                const BN_ULONG *np, const BN_ULONG *n0, int num, int power) {
+  perror("bn_powerx5");
+  abort();
+}
+
+int bn_sqr8x_mont(BN_ULONG *rp, const BN_ULONG *ap, BN_ULONG mulx_adx_capable,
+                  const BN_ULONG *np, const BN_ULONG *n0, size_t num) {
+  perror("bn_sqr8x_mont");
+  abort();
+}
+#endif
+
+#endif // defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX)