add fix for issue #17

1 files changed, 148 insertions, 7 deletions

diff --git a/src/openssl.c b/src/openssl.c
index 3465922..163ba7f 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -33,7 +33,9 @@
 #include <math.h>         /* INFINITY fabs(3) floor(3) frexp(3) fmod(3) round(3) isfinite(3) */
 #include <time.h>         /* struct tm time_t strptime(3) time(2) */
 #include <ctype.h>        /* tolower(3) */
+#include <signal.h>       /* sig_atomic_t */
 #include <errno.h>        /* ENOMEM errno */
+#include <assert.h>       /* assert */
 
 #include <sys/types.h>    /* ssize_t pid_t */
 #if !defined __sun && !defined _AIX
@@ -44,16 +46,11 @@
 #include <sys/socket.h>   /* AF_INET AF_INET6 */
 #include <sys/resource.h> /* RUSAGE_SELF struct rusage getrusage(2) */
 #include <sys/utsname.h>  /* struct utsname uname(3) */
-
 #include <fcntl.h>        /* O_RDONLY O_CLOEXEC open(2) */
-
 #include <unistd.h>       /* close(2) getpid(2) */
-
 #include <netinet/in.h>   /* struct in_addr struct in6_addr */
 #include <arpa/inet.h>    /* inet_pton(3) */
-
 #include <pthread.h>      /* pthread_mutex_init(3) pthread_mutex_lock(3) pthread_mutex_unlock(3) */
-
 #include <dlfcn.h>        /* dladdr(3) dlopen(3) */
 
 #if __APPLE__
@@ -576,6 +573,22 @@ static void lib_setintegers(lua_State *L, const integer_Reg *l) {
 } /* lib_setintegers() */
 
 
+#define COMPAT_X509_STORE_FREE_BUG 0x01
+
+static struct {
+	int flags;
+
+	int SSL_CTX_ex_index;
+	void (*X509_STORE_free)(X509_STORE *);
+
+	struct {
+		X509_STORE *store;
+	} tmp;
+} compat = {
+	.flags = 0,
+	.SSL_CTX_ex_index = -1,
+	.X509_STORE_free = &X509_STORE_free,
+};
 
 #if !HAVE_EVP_PKEY_base_id
 #define EVP_PKEY_base_id(key) compat_EVP_PKEY_base_id((key))
@@ -621,6 +634,126 @@ static void *compat_EVP_PKEY_get0(EVP_PKEY *key) {
 } /* compat_EVP_PKEY_get0() */
 #endif
 
+/* X509_STORE_free in OpenSSL versions < 1.0.2 doesn't obey reference count */
+#define X509_STORE_free(store) \
+	(compat.X509_STORE_free)((store))
+
+static void compat_X509_STORE_free(X509_STORE *store) {
+	int i;
+
+	i = CRYPTO_add(&store->references, -1, CRYPTO_LOCK_X509_STORE);
+
+        if (i > 0)
+                return;
+
+	(X509_STORE_free)(store);
+} /* compat_X509_STORE_free() */
+
+#if !HAVE_SSL_CTX_set1_cert_store
+#define SSL_CTX_set1_cert_store(ctx, store) \
+	compat_SSL_CTX_set1_cert_store((ctx), (store))
+
+static void compat_SSL_CTX_set1_cert_store(SSL_CTX *ctx, X509_STORE *store) {
+	int n;
+
+	/*
+	 * This isn't thead-safe, but using X509_STORE or SSL_CTX objects
+	 * from different threads isn't safe generally.
+	 */
+	if (ctx->cert_store) {
+		X509_STORE_free(ctx->cert_store);
+		ctx->cert_store = NULL;
+	}
+
+	n = store->references;
+
+	SSL_CTX_set_cert_store(ctx, store);
+
+	if (n == store->references)
+		CRYPTO_add(&store->references, 1, CRYPTO_LOCK_X509_STORE);
+} /* compat_SSL_CTX_set1_cert_store() */
+#endif
+
+static void compat_SSL_CTX_onfree(void *_ctx, void *data NOTUSED, CRYPTO_EX_DATA *ad NOTUSED, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
+	SSL_CTX *ctx = _ctx;
+
+	if (ctx->cert_store) {
+		X509_STORE_free(ctx->cert_store);
+		ctx->cert_store = NULL;
+	}
+} /* compat_SSL_CTX_onfree() */
+
+/* helper routine to determine if X509_STORE_free obeys reference count */
+static void compat_init_X509_STORE_onfree(void *store, void *data NOTUSED, CRYPTO_EX_DATA *ad NOTUSED, int idx NOTUSED, long argl NOTUSED, void *argp NOTUSED) {
+	/* unfortunately there's no way to remove a handler */
+	if (store != compat.tmp.store)
+		return;
+
+	/* signal that we were freed by nulling our reference */
+	compat.tmp.store = NULL;
+} /* compat_init_X509_STORE_onfree() */
+
+static int compat_init(void) {
+	static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+	static volatile sig_atomic_t done;
+	int error;
+
+	if ((error = pthread_mutex_lock(&mutex)))
+		return error;
+
+	if (!done) {
+		/*
+		 * Test if X509_STORE_free obeys reference counts.
+		 */
+		if (-1 == CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_X509_STORE, 0, NULL, NULL, NULL, &compat_init_X509_STORE_onfree))
+			goto sslerr;
+
+		if (!(compat.tmp.store = X509_STORE_new()))
+			goto sslerr;
+
+		CRYPTO_add(&compat.tmp.store->references, 1, CRYPTO_LOCK_X509_STORE);
+		X509_STORE_free(compat.tmp.store);
+
+		if (compat.tmp.store) {
+			X509_STORE_free(compat.tmp.store);
+			assert(compat.tmp.store == NULL);
+			compat.tmp.store = NULL;
+		} else {
+			/*
+			 * If X509_STORE_free does NOT obey reference
+			 * counts, then make sure that our fixed version is
+			 * called on SSL_CTX destruction. Note that this
+			 * won't fix code which doesn't properly obey the
+			 * reference counts when setting the cert_store
+			 * member.
+			 */
+			if (-1 == CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_SSL_CTX, 0, NULL, NULL, NULL, &compat_SSL_CTX_onfree))
+				goto sslerr;
+
+			compat.flags |= COMPAT_X509_STORE_FREE_BUG;
+		}
+
+		done = 1;
+	}
+epilog:
+	if (compat.tmp.store) {
+		X509_STORE_free(compat.tmp.store);
+		compat.tmp.store = NULL;
+	}
+
+	pthread_mutex_unlock(&mutex);
+
+	return 0;
+sslerr:
+	error = -1;
+
+	goto epilog;
+syserr:
+	error = errno;
+
+	goto epilog;
+} /* compat_init() */
+
 
 typedef int auxref_t;
 typedef int auxtype_t;
@@ -4699,8 +4832,7 @@ static int sx_setStore(lua_State *L) {
 	SSL_CTX *ctx = checksimple(L, 1, SSL_CTX_CLASS);
 	X509_STORE *store = checksimple(L, 2, X509_STORE_CLASS);
 
-	SSL_CTX_set_cert_store(ctx, store);
-	CRYPTO_add(&store->references, 1, CRYPTO_LOCK_X509_STORE);
+	SSL_CTX_set1_cert_store(ctx, store);
 
 	lua_pushboolean(L, 1);
 
@@ -6251,6 +6383,15 @@ static void initall(lua_State *L) {
 		}
 	}
 
+	/* TODO: Move down to after SSL_load_error_strings */
+	if ((error = compat_init())) {
+		if (error == -1) {
+			throwssl(L, "openssl.init");
+		} else {
+			luaL_error(L, "openssl.init: %s", xstrerror(error));
+		}
+	}
+
 	pthread_mutex_lock(&mutex);
 
 	if (!initssl) {