refactor xc_signature to auxL_optdigest

refactor xc_getPublicKeyDigestName to use auxL_optdigest
closes issue #63

1 files changed, 116 insertions, 29 deletions

diff --git a/src/openssl.c b/src/openssl.c
index fd392d7..ed88c3a 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -77,12 +77,34 @@
 #include "compat52.h"
 #endif
 
+#define GNUC_2VER(M, m, p) (((M) * 10000) + ((m) * 100) + (p))
+#define GNUC_PREREQ(M, m, p) (__GNUC__ > 0 && GNUC_2VER(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) >= GNUC_2VER((M), (m), (p)))
+
+#define MSC_2VER(M, m, p) ((((M) + 6) * 10000000) + ((m) * 1000000) + (p))
+#define MSC_PREREQ(M, m, p) (_MSC_VER_FULL > 0 && _MSC_VER_FULL >= MSC_2VER((M), (m), (p)))
+
 #define OPENSSL_PREREQ(M, m, p) \
 	(OPENSSL_VERSION_NUMBER >= (((M) << 28) | ((m) << 20) | ((p) << 12)) && !defined LIBRESSL_VERSION_NUMBER)
 
 #define LIBRESSL_PREREQ(M, m, p) \
 	(LIBRESSL_VERSION_NUMBER >= (((M) << 28) | ((m) << 20) | ((p) << 12)))
 
+#ifndef __has_builtin
+#define __has_builtin(x) 0
+#endif
+
+#ifndef __has_extension
+#define __has_extension(x) 0
+#endif
+
+#ifndef HAVE___ASSUME
+#define HAVE___ASSUME MSC_PREREQ(8,0,0)
+#endif
+
+#ifndef HAVE___BUILTIN_UNREACHABLE
+#define HAVE___BUILTIN_UNREACHABLE (GNUC_PREREQ(4,5,0) || __has_builtin(__builtin_unreachable))
+#endif
+
 #ifndef HAVE_ASN1_STRING_GET0_DATA
 #define HAVE_ASN1_STRING_GET0_DATA OPENSSL_PREREQ(1,1,0)
 #endif
@@ -320,6 +342,13 @@
 #define NOTUSED
 #endif
 
+#if HAVE___BUILTIN_UNREACHABLE
+#define NOTREACHED __builtin_unreachable()
+#elif HAVE___ASSUME
+#define NOTREACHED __assume(0)
+#else
+#define NOTREACHED (void)0
+#endif
 
 #define countof(a) (sizeof (a) / sizeof *(a))
 #define endof(a) (&(a)[countof(a)])
@@ -714,6 +743,23 @@ static size_t auxS_obj2txt(void *dst, size_t lim, const ASN1_OBJECT *obj) {
 	return auxS_obj2id(dst, lim, obj);
 } /* auxS_obj2txt() */
 
+static const EVP_MD *auxS_todigest(const char *name, EVP_PKEY *key, const EVP_MD *def) {
+	const EVP_MD *md;
+	int nid;
+
+	if (name) {
+		if ((md = EVP_get_digestbyname(name)))
+			return md;
+	} else if (key) {
+		if ((EVP_PKEY_get_default_digest_nid(key, &nid) > 0)) {
+			if ((md = EVP_get_digestbynid(nid)))
+				return md;
+		}
+	}
+
+	return def;
+} /* auxS_todigest() */
+
 static _Bool auxS_isoid(const char *txt) {
 	return (*txt >= '0' && *txt <= '9');
 } /* auxS_isoid() */
@@ -753,6 +799,11 @@ static _Bool auxS_txt2nid(int *nid, const char *txt) {
  *
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+static int auxL_absindex(lua_State *L, int *index) {
+	*index = lua_absindex(L, *index);
+	return *index;
+} /* auxL_absindex() */
+
 typedef int auxref_t;
 typedef int auxtype_t;
 
@@ -1097,8 +1148,9 @@ static const char *auxL_pusherror(lua_State *L, int error, const char *fun) {
 
 static int auxL_error(lua_State *L, int error, const char *fun) {
 	auxL_pusherror(L, error, fun);
-
-	return lua_error(L);
+	lua_error(L);
+	NOTREACHED;
+	return 0;
 } /* auxL_error() */
 
 static const char *auxL_pushnid(lua_State *L, int nid) {
@@ -1113,6 +1165,24 @@ static const char *auxL_pushnid(lua_State *L, int nid) {
 	return lua_tostring(L, -1);
 } /* auxL_pushnid() */
 
+static const EVP_MD *auxL_optdigest(lua_State *L, int index, EVP_PKEY *key, const EVP_MD *def) {
+	const char *name = luaL_optstring(L, auxL_absindex(L, &index), NULL);
+	const EVP_MD *md;
+
+	if ((md = auxS_todigest(name, key, NULL)))
+		return md;
+
+	if (name) {
+		luaL_argerror(L, index, lua_pushfstring(L, "invalid digest type (%s)", name));
+		NOTREACHED;
+	} else if (key) {
+		luaL_argerror(L, index, lua_pushfstring(L, "no digest type for key type (%d)", EVP_PKEY_base_id(key)));
+		NOTREACHED;
+	}
+
+	return def;
+} /* auxL_optdigest() */
+
 
 /*
  * dl - dynamically loaded module management
@@ -3400,18 +3470,11 @@ static int pk_toPEM(lua_State *L) {
 static int pk_getDefaultDigestName(lua_State *L) {
 	EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
 	int nid;
-	char txt[256];
-	size_t len;
 
 	if (!(EVP_PKEY_get_default_digest_nid(key, &nid) > 0))
 		return auxL_error(L, auxL_EOPENSSL, "pkey:getDefaultDigestName");
 
-	if (!(len = auxS_nid2txt(txt, sizeof txt, nid)))
-		return auxL_error(L, auxL_EOPENSSL, "pkey:getDefaultDigestName");
-	if (len > sizeof txt)
-		return auxL_error(L, EOVERFLOW, "pkey:getDefaultDigestName");
-
-	lua_pushlstring(L, txt, len);
+	auxL_pushnid(L, nid);
 
 	return 1;
 } /* pk_getDefaultDigestName() */
@@ -5730,6 +5793,7 @@ static int xc_setPublicKey(lua_State *L) {
 } /* xc_setPublicKey() */
 
 
+#if 0
 static int xc_getPublicKeyDigest(lua_State *L) {
 	ASN1_BIT_STRING *pk = X509_get0_pubkey_bitstr(checksimple(L, 1, X509_CERT_CLASS));
 	const char *id = luaL_optstring(L, 2, "sha1");
@@ -5747,33 +5811,53 @@ static int xc_getPublicKeyDigest(lua_State *L) {
 
 	return 1;
 } /* xc_getPublicKeyDigest() */
+#else
+static int xc_getPublicKeyDigest(lua_State *L) {
+	X509 *crt = checksimple(L, 1, X509_CERT_CLASS);
+	EVP_PKEY *key;
+	const EVP_MD *md;
+	ASN1_BIT_STRING *bitstr;
+	unsigned char digest[EVP_MAX_MD_SIZE];
+	unsigned int len;
 
+	if (!(key = X509_get_pubkey(crt)))
+		return luaL_argerror(L, 1, "no public key");
+	md = auxL_optdigest(L, 2, key, NULL);
+	bitstr = X509_get0_pubkey_bitstr(crt);
 
-static const EVP_MD *xc_signature(lua_State *L, int index, EVP_PKEY *key) {
-	const char *id;
-	const EVP_MD *md;
+	if (!EVP_Digest(bitstr->data, bitstr->length, digest, &len, md, NULL))
+		return auxL_error(L, auxL_EOPENSSL, "x509.cert:getPublicKeyDigest");
+	lua_pushlstring(L, (char *)digest, len);
+
+	return 1;
+} /* xc_getPublicKeyDigest() */
+#endif
+
+
+#if 0
+/*
+ * TODO: X509_get_signature_type always seems to return NID_undef. Are we
+ * using it wrong or is it broken?
+ */
+static int xc_getSignatureName(lua_State *L) {
+	X509 *crt = checksimple(L, 1, X509_CERT_CLASS);
 	int nid;
 
-	if ((id = luaL_optstring(L, index, NULL))) {
-		if (!(md = EVP_get_digestbyname(id)))
-			goto unknown;
-	} else {
-		if (!(EVP_PKEY_get_default_digest_nid(key, &nid) > 0))
-			goto unknown;
-		if (!(md = EVP_get_digestbynid(nid)))
-			goto unknown;
-	}
+	if (NID_undef == (nid = X509_get_signature_type(crt)))
+		return 0;
+
+	auxL_pushnid(L, nid);
+
+	return 1;
+} /* xc_getSignatureName() */
+#endif
 
-	return md;
-unknown:
-	return EVP_sha1();
-} /* xc_signature() */
 
 static int xc_sign(lua_State *L) {
 	X509 *crt = checksimple(L, 1, X509_CERT_CLASS);
 	EVP_PKEY *key = checksimple(L, 2, PKEY_CLASS);
 
-	if (!X509_sign(crt, key, xc_signature(L, 3, key)))
+	if (!X509_sign(crt, key, auxL_optdigest(L, 3, key, NULL)))
 		return auxL_error(L, auxL_EOPENSSL, "x509.cert:sign");
 
 	lua_pushboolean(L, 1);
@@ -5913,6 +5997,9 @@ static const auxL_Reg xc_methods[] = {
 	{ "getPublicKey",  &xc_getPublicKey },
 	{ "setPublicKey",  &xc_setPublicKey },
 	{ "getPublicKeyDigest", &xc_getPublicKeyDigest },
+#if 0
+	{ "getSignatureName", &xc_getSignatureName },
+#endif
 	{ "sign",          &xc_sign },
 	{ "text",          &xc_text },
 	{ "tostring",      &xc__tostring },
@@ -6166,7 +6253,7 @@ static int xr_sign(lua_State *L) {
 	X509_REQ *csr = checksimple(L, 1, X509_CSR_CLASS);
 	EVP_PKEY *key = checksimple(L, 2, PKEY_CLASS);
 
-	if (!X509_REQ_sign(csr, key, xc_signature(L, 3, key)))
+	if (!X509_REQ_sign(csr, key, auxL_optdigest(L, 3, key, NULL)))
 		return auxL_error(L, auxL_EOPENSSL, "x509.csr:sign");
 
 	lua_pushboolean(L, 1);
@@ -6541,7 +6628,7 @@ static int xx_sign(lua_State *L) {
 	X509_CRL *crl = checksimple(L, 1, X509_CRL_CLASS);
 	EVP_PKEY *key = checksimple(L, 2, PKEY_CLASS);
 
-	if (!X509_CRL_sign(crl, key, xc_signature(L, 3, key)))
+	if (!X509_CRL_sign(crl, key, auxL_optdigest(L, 3, key, NULL)))
 		return auxL_error(L, auxL_EOPENSSL, "x509.crl:sign");
 
 	lua_pushboolean(L, 1);