Add ssl_pushsafe function to avoid panic on OOM

1 files changed, 41 insertions, 7 deletions

diff --git a/src/openssl.c b/src/openssl.c
index 35ddaf7..504931d 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -682,9 +682,43 @@ static void *loadfield_udata(lua_State *L, int index, const char *k, const char
 } /* loadfield_udata() */
 
 
-/* Forward declarations */
+/* Forward declaration */
 static SSL *ssl_push(lua_State *, SSL *);
 
+/* push an ssl object into lua in a way that is safe from OOM
+ * Lua 5.1 does not support normally returning values from lua_cpcall
+ * to return a value, we instead return it via an error object
+ */
+static int ssl_pushsafe_helper(lua_State *L) {
+	ssl_push(L, lua_touserdata(L, 1));
+#if LUA_VERSION_NUM <= 501
+	return lua_error(L);
+#else
+	return 1;
+#endif
+}
+
+static int ssl_pushsafe(lua_State *L, SSL *ssl) {
+	int status;
+#if LUA_VERSION_NUM <= 501
+	status = lua_cpcall(L, ssl_pushsafe_helper, ssl);
+	if (status == LUA_ERRRUN)
+		status = LUA_OK;
+	else if (status == LUA_OK)
+		/* this should be impossible */
+		status = LUA_ERRRUN;
+	else
+		lua_pop(L, 1);
+#else
+	lua_pushcfunction(L, ssl_pushsafe_helper);
+	lua_pushlightuserdata(L, ssl);
+	status = lua_pcall(L, 1, 1, 0);
+	if (status != LUA_OK)
+		lua_pop(L, 1);
+#endif
+	return status;
+}
+
 
 /*
  * Auxiliary C routines
@@ -8057,12 +8091,12 @@ static int sx_setAlpnSelect_cb(SSL *ssl, const unsigned char **out, unsigned cha
 
 	otop = lua_gettop(L) - n;
 
-	/* TODO: Install temporary panic handler to catch OOM errors */
-
 	/* pass SSL object as 1st argument */
-	ssl_push(L, ssl);
+	if (ssl_pushsafe(L, ssl))
+		goto fatal;
 	lua_insert(L, otop + 3);
 
+	/* TODO: Install temporary panic handler to catch OOM errors */
 	/* pass table of protocol names as 2nd argument */
 	pushprotos(L, in, inlen);
 	lua_insert(L, otop + 4);
@@ -8149,10 +8183,10 @@ static int sx_setHostnameCallback_cb(SSL *ssl, int *ad, void *_ctx) {
 
 	otop = lua_gettop(L) - n;
 
-	/* TODO: Install temporary panic handler to catch OOM errors */
-
 	/* pass SSL object as 1st argument */
-	ssl_push(L, ssl);
+	if (ssl_pushsafe(L, ssl))
+		goto done;
+
 	lua_insert(L, otop + 2);
 
 	if (LUA_OK != (status = lua_pcall(L, 1 + (n - 1), 2, 0)))