Commit 390555f31f7e1b436decbc9e90e41d65ebedc4c1

czurnieden 2019-01-16T22:45:01

Added overflow check in mp_prime_is_prime and some verbosity in demo.c

diff --git a/bn_mp_prime_is_prime.c b/bn_mp_prime_is_prime.c
index 63d3725..8d96e6a 100644
--- a/bn_mp_prime_is_prime.c
+++ b/bn_mp_prime_is_prime.c
@@ -305,8 +305,12 @@ int mp_prime_is_prime(const mp_int *a, int t, int *result)
             fips_rand &= mask;
          }
 #endif
-         /* Ceil, because small numbers have a right to live, too, */
-         len = (((int)fips_rand + DIGIT_BIT) / DIGIT_BIT);
+         if (fips_rand > ((unsigned int) INT_MAX - DIGIT_BIT)) {
+            len = INT_MAX / DIGIT_BIT;
+         }
+         else {
+            len = (((int)fips_rand + DIGIT_BIT) / DIGIT_BIT);
+         }
          /*  Unlikely. */
          if (len < 0) {
             ix--;
diff --git a/demo/demo.c b/demo/demo.c
index 0aeae25..2a65601 100644
--- a/demo/demo.c
+++ b/demo/demo.c
@@ -687,9 +687,21 @@ int main(void)
    mp_set(&a,1u);
    mp_mul_2d(&a,1119,&a);
    mp_add_d(&a,53,&a);
-   mp_prime_is_prime(&a, 8, &cnt);
+   err = mp_prime_is_prime(&a, 8, &cnt);
+   /* small problem */
+   if (err != MP_OKAY) {
+      printf("failed with err code %d\n", err);
+      printf("prime tested was: ");
+      mp_fwrite(&a,16,stdout);
+      putchar('\n');
+      return EXIT_FAILURE;
+   }
+   /* large problem */
    if (cnt == MP_NO) {
-      printf("A certified prime is a prime but mp_prime_is_prime says it not.\n");
+      printf("A certified prime is a prime but mp_prime_is_prime says it is not.\n");
+      printf("prime tested was: ");
+      mp_fwrite(&a,16,stdout);
+      putchar('\n');
       return EXIT_FAILURE;
    }
    for (ix = 16; ix < 128; ix++) {
@@ -709,19 +721,42 @@ int main(void)
       /* let's see if it's really a safe prime */
       mp_sub_d(&a, 1uL, &a);
       mp_div_2(&a, &a);
-      mp_prime_is_prime(&a, 8, &cnt);
+      err = mp_prime_is_prime(&a, 8, &cnt);
+      /* small problem */
+      if (err != MP_OKAY) {
+         printf("failed with err code %d\n", err);
+         printf("prime tested was: ");
+         mp_fwrite(&a,16,stdout);
+         putchar('\n');
+         return EXIT_FAILURE;
+      }
+      /* large problem */
       if (cnt != MP_YES) {
          printf("sub is not prime!\n");
+         printf("prime failed was: ");
+         mp_fwrite(&a,16,stdout);
+         putchar('\n');
          return EXIT_FAILURE;
       }
+
    }
    /* Check regarding problem #143 */
 #ifndef MP_8BIT
    mp_read_radix(&a,    "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A63A3620FFFFFFFFFFFFFFFF",
                  16);
-   mp_prime_strong_lucas_selfridge(&a, &cnt);
+   err = mp_prime_strong_lucas_selfridge(&a, &cnt);
+   if (err != MP_OKAY) {
+      printf("mp_prime_strong_lucas_selfridge failed with err code %d\n", err);
+      printf("prime tested was: ");
+      mp_fwrite(&a,16,stdout);
+      putchar('\n');
+      return EXIT_FAILURE;
+   }
    if (cnt != MP_YES) {
       printf("\n\nissue #143 - mp_prime_strong_lucas_selfridge FAILED!\n");
+      printf("prime tested was: ");
+      mp_fwrite(&a,16,stdout);
+      putchar('\n');
       return EXIT_FAILURE;
    }
 #endif