bignum: validate hex and decimal strings before feeding to openssl

OpenSSL doesn't throw an error on invalid numbers

1 files changed, 11 insertions, 2 deletions

diff --git a/src/openssl.c b/src/openssl.c
index 66a6168..4ca8da7 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -29,7 +29,7 @@
 #include <strings.h>      /* strcasecmp(3) */
 #include <math.h>         /* INFINITY fabs(3) floor(3) frexp(3) fmod(3) round(3) isfinite(3) */
 #include <time.h>         /* struct tm time_t strptime(3) time(2) */
-#include <ctype.h>        /* tolower(3) */
+#include <ctype.h>        /* isdigit(3), isxdigit(3), tolower(3) */
 #include <errno.h>        /* ENOMEM ENOTSUP EOVERFLOW errno */
 #include <assert.h>       /* assert */
 
@@ -1685,7 +1685,7 @@ static _Bool f2bn(BIGNUM **bn, double f) {
 static BIGNUM *(checkbig)(lua_State *L, int index, _Bool *lvalue) {
 	BIGNUM **bn;
 	const char *str;
-	size_t len;
+	size_t len, i;
 	_Bool neg, hex = 0;
 
 	index = lua_absindex(L, index);
@@ -1702,6 +1702,15 @@ static BIGNUM *(checkbig)(lua_State *L, int index, _Bool *lvalue) {
 
 		if (str[neg] == '0' && (str[neg+1] == 'x' || str[neg+1] == 'X')) {
 			hex = 1;
+			for (i = 2+neg; i < len; i++) {
+				if (!isxdigit(str[i]))
+					luaL_argerror(L, 1, "invalid hex string");
+			}
+		} else {
+			for (i = neg; i < len; i++) {
+				if (!isdigit(str[i]))
+					luaL_argerror(L, 1, "invalid decimal string");
+			}
 		}
 
 		bn = prepsimple(L, BIGNUM_CLASS);