From 2391a59e43de4ccc685b7790d84a9a4705750dd3 Mon Sep 17 00:00:00 2001
From: William Ahern <william@25thandclement.com>
Date: Sat, 29 Oct 2016 13:48:07 -0700
Subject: use EVP_PKEY_get_default_digest_nid to determine the default
 signature, and emulate for OpenSSL 0.9.8 using our old code from xc_signature

---
 src/openssl.c | 52 +++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 11 deletions(-)

diff --git a/src/openssl.c b/src/openssl.c
index 8208f78..f5d9343 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -119,6 +119,10 @@
 #define HAVE_DSA_SET0_PQG OPENSSL_PREREQ(1,1,0)
 #endif
 
+#ifndef HAVE_EVP_PKEY_GET_DEFAULT_DIGEST_NID
+#define HAVE_EVP_PKEY_GET_DEFAULT_DIGEST_NID OPENSSL_PREREQ(0,9,9)
+#endif
+
 #ifndef HAVE_EVP_PKEY_BASE_ID
 #define HAVE_EVP_PKEY_BASE_ID OPENSSL_PREREQ(1,1,0)
 #endif
@@ -1232,6 +1236,29 @@ static int compat_EVP_PKEY_base_id(EVP_PKEY *key) {
 } /* compat_EVP_PKEY_base_id() */
 #endif
 
+#if !HAVE_EVP_PKEY_GET_DEFAULT_DIGEST_NID
+#define EVP_PKEY_get_default_digest_nid(...) \
+	compat_EVP_PKEY_get_default_digest_nid(__VA_ARGS__)
+
+static int compat_EVP_PKEY_get_default_digest_nid(EVP_PKEY *key, int *nid) {
+	switch (EVP_PKEY_base_id(key)) {
+	case EVP_PKEY_RSA:
+		*nid = EVP_MD_nid(EVP_sha1());
+		break;
+	case EVP_PKEY_DSA:
+		*nid = EVP_MD_nid(EVP_dss1());
+		break;
+	case EVP_PKEY_EC:
+		*nid = EVP_MD_nid(EVP_ecdsa());
+		break;
+	default:
+		*nid = EVP_MD_nid(EVP_md_null());
+		break;
+	}
+
+	return 1;
+} /* compat_EVP_PKEY_get_default_digest_nid() */
+#endif
 
 #if !HAVE_EVP_PKEY_GET0
 #define EVP_PKEY_get0(key) compat_EVP_PKEY_get0((key))
@@ -5559,20 +5586,23 @@ static int xc_getPublicKeyDigest(lua_State *L) {
 static const EVP_MD *xc_signature(lua_State *L, int index, EVP_PKEY *key) {
 	const char *id;
 	const EVP_MD *md;
+	int nid;
 
-	if ((id = luaL_optstring(L, index, NULL)))
-		return ((md = EVP_get_digestbyname(id)))? md : EVP_md_null();
+	if ((id = luaL_optstring(L, index, NULL))) {
+		if (!(md = EVP_get_digestbyname(id)))
+			goto unknown;
 
-	switch (EVP_PKEY_base_id(key)) {
-	case EVP_PKEY_RSA:
-		return EVP_sha1();
-	case EVP_PKEY_DSA:
-		return EVP_dss1();
-	case EVP_PKEY_EC:
-		return EVP_ecdsa();
-	default:
-		return EVP_md_null();
+		return md;
 	}
+
+	if (!(EVP_PKEY_get_default_digest_nid(key, &nid) > 0))
+		goto unknown;
+	if (!(md = EVP_get_digestbynid(nid)))
+		goto unknown;
+
+	return md;
+unknown:
+	return EVP_md_null();
 } /* xc_signature() */
 
 static int xc_sign(lua_State *L) {
-- 
cgit v1.2.3-59-g8ed1b