add DER input and output formats, plus 'autodetect' PEM-or-DER, for almost everything (except pk__tostring and pk_toPEM, FIXME)

1 files changed, 167 insertions, 62 deletions

diff --git a/src/openssl.c b/src/openssl.c
index 1554053..84d8079 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -31,6 +31,7 @@
 #include <strings.h>	/* strcasecmp(3) */
 #include <math.h>	/* INFINITY fabs(3) floor(3) frexp(3) fmod(3) round(3) isfinite(3) */
 #include <time.h>	/* struct tm time_t strptime(3) */
+#include <ctype.h>      /* tolower(3) */
 
 #include <sys/types.h>
 #include <sys/stat.h>	/* struct stat stat(2) */
@@ -96,6 +97,8 @@
 #define stricmp(a, b) strcasecmp((a), (b))
 #define strieq(a, b) (!stricmp((a), (b)))
 
+#define xtolower(c) tolower((unsigned char)(c))
+
 #define SAY_(file, func, line, fmt, ...) \
 	fprintf(stderr, "%s:%d: " fmt "%s", __func__, __LINE__, __VA_ARGS__)
 
@@ -218,7 +221,7 @@ static void addclass(lua_State *L, const char *name, const luaL_Reg *methods, co
 } /* addclass() */
 
 
-static int checkoption(struct lua_State *L, int index, const char *def, const char *opts[]) {
+static int checkoption(struct lua_State *L, int index, const char *def, const char *const opts[]) {
 	const char *opt = (def)? luaL_optstring(L, index, def) : luaL_checkstring(L, index);
 	int i; 
 
@@ -231,6 +234,34 @@ static int checkoption(struct lua_State *L, int index, const char *def, const ch
 } /* checkoption() */
 
 
+#define X509_ANY 0x01
+#define X509_PEM 0x02
+#define X509_DER 0x04
+#define X509_ALL (X509_PEM|X509_DER)
+
+static int optencoding(lua_State *L, int index, const char *def, int allow) {
+	static const char *const opts[] = { "*", "pem", "der", NULL };
+	int type = 0;
+
+	switch (checkoption(L, index, def, opts)) {
+	case 0:
+		type = X509_ANY;
+		break;
+	case 1:
+		type = X509_PEM;
+		break;
+	case 2:
+		type = X509_DER;
+		break;
+	}
+
+	if (!(type & allow))
+		luaL_argerror(L, index, lua_pushfstring(L, "invalid option %s", luaL_checkstring(L, index)));
+
+	return type;
+} /* optencoding() */
+
+
 static _Bool getfield(lua_State *L, int index, const char *k) {
 	lua_getfield(L, index, k);
 
@@ -719,11 +750,12 @@ static BIO *getbio(lua_State *L) {
 static int pk_new(lua_State *L) {
 	EVP_PKEY **ud;
 
-	lua_settop(L, 1);
+	/* #1 table or key; if key, #2 format and #3 type */
+	lua_settop(L, 3);
 
 	ud = prepsimple(L, PUBKEY_CLASS);
 
-	if (lua_istable(L, 1)) {
+	if (lua_istable(L, 1) || lua_isnil(L, 1)) {
 		int type = EVP_PKEY_RSA;
 		unsigned bits = 1024;
 		unsigned exp = 65537;
@@ -860,37 +892,69 @@ creat:
 		default:
 			return luaL_error(L, "%d: unknown EVP base type (%d)", EVP_PKEY_type(type), type);
 		} /* switch() */
-	} else {
-		const char *pem;
+	} else if (lua_isstring(L, 1)) {
+		int type = optencoding(L, 2, "*", X509_ANY|X509_PEM|X509_DER);
+		int ispub = -1;
+		const char *opt, *data;
 		size_t len;
 		BIO *bio;
-		int ok;
+		int ok = 0;
+
+		/* check if specified publickey or privatekey */
+		if ((opt = luaL_optstring(L, 3, NULL))) {
+			if (xtolower(opt[0]) == 'p' && xtolower(opt[1]) == 'u') {
+				ispub = 1;
+			} else if (xtolower(opt[0]) == 'p' && xtolower(opt[1]) == 'r') {
+				ispub = 0;
+			} else {
+				return luaL_argerror(L, 3, lua_pushfstring(L, "invalid option %s", opt));
+			}
+		}
 
-		if (!(*ud = EVP_PKEY_new()))
+		data = luaL_checklstring(L, 1, &len);
+
+		if (!(bio = BIO_new_mem_buf((void *)data, len)))
 			return throwssl(L, "pubkey.new");
 
-		switch (lua_type(L, 1)) {
-		case LUA_TSTRING:
-			pem = luaL_checklstring(L, 1, &len);
+		if (type == X509_PEM || type == X509_ANY) {
+			if (ispub == 1 || ispub == -1) {
+				ok = !!(*ud = PEM_read_bio_PUBKEY(bio, NULL, 0, ""));
 
-			if (!(bio = BIO_new_mem_buf((void *)pem, len)))
-				return throwssl(L, "pubkey.new");
+				if (ok || (type == X509_PEM && ispub == 1))
+					goto done;
+			}
 
-			if (strstr(pem, "PUBLIC KEY")) {
-				ok = !!PEM_read_bio_PUBKEY(bio, ud, 0, 0);
-			} else {
-				ok = !!PEM_read_bio_PrivateKey(bio, ud, 0, 0);
+			if (ispub == 0 || ispub == -1) {
+				ok = !!(*ud = PEM_read_bio_PrivateKey(bio, NULL, 0, ""));
+
+				if (ok || (type == X509_PEM && ispub == 0))
+					goto done;
 			}
+		}
 
-			BIO_free(bio);
+		if (type == X509_DER || type == X509_ANY) {
+			if (ispub == 1 || ispub == -1) {
+				ok = !!(*ud = d2i_PUBKEY_bio(bio, NULL));
 
-			if (!ok)
-				return throwssl(L, "pubkey.new");
+				if (ok || (type == X509_DER && ispub == 1))
+					goto done;
+			}
 
-			break;
-		default:
-			return luaL_error(L, "%s: unknown key initializer", lua_typename(L, lua_type(L, 1)));
-		} /* switch() */
+			if (ispub == 0 || ispub == -1) {
+				ok = !!(*ud = d2i_PrivateKey_bio(bio, NULL));
+
+				if (ok || (type == X509_DER && ispub == 0))
+					goto done;
+			}
+		}
+
+done:
+		BIO_free(bio);
+
+		if (!ok)
+			return throwssl(L, "pubkey.new");
+	} else {
+		return luaL_error(L, "%s: unknown key initializer", lua_typename(L, lua_type(L, 1)));
 	}
 
 	return 1;
@@ -914,19 +978,24 @@ static int pk_type(lua_State *L) {
 
 static int pk_setPublicKey(lua_State *L) {
 	EVP_PKEY **key = luaL_checkudata(L, 1, PUBKEY_CLASS);
-	const char *pem;
+	const char *data;
 	size_t len;
 	BIO *bio;
-	int ok;
-
-	lua_settop(L, 2);
+	int type, ok = 0;
 
-	pem = luaL_checklstring(L, 2, &len);
+	data = luaL_checklstring(L, 2, &len);
+	type = optencoding(L, 3, "*", X509_ANY|X509_PEM|X509_DER);
 
-	if (!(bio = BIO_new_mem_buf((void *)pem, len)))
+	if (!(bio = BIO_new_mem_buf((void *)data, len)))
 		return throwssl(L, "pubkey.new");
 
-	ok = !!PEM_read_bio_PUBKEY(bio, key, 0, 0);
+	if (type == X509_ANY || type == X509_PEM) {
+		ok = !!PEM_read_bio_PUBKEY(bio, key, 0, "");
+	}
+
+	if (!ok && (type == X509_ANY || type == X509_DER)) {
+		ok = !!d2i_PUBKEY_bio(bio, key);
+	}
 
 	BIO_free(bio);
 
@@ -941,19 +1010,24 @@ static int pk_setPublicKey(lua_State *L) {
 
 static int pk_setPrivateKey(lua_State *L) {
 	EVP_PKEY **key = luaL_checkudata(L, 1, PUBKEY_CLASS);
-	const char *pem;
+	const char *data;
 	size_t len;
 	BIO *bio;
-	int ok;
+	int type, ok = 0;
 
-	lua_settop(L, 2);
+	data = luaL_checklstring(L, 2, &len);
+	type = optencoding(L, 3, "*", X509_ANY|X509_PEM|X509_DER);
 
-	pem = luaL_checklstring(L, 2, &len);
-
-	if (!(bio = BIO_new_mem_buf((void *)pem, len)))
+	if (!(bio = BIO_new_mem_buf((void *)data, len)))
 		return throwssl(L, "pubkey.new");
 
-	ok = !!PEM_read_bio_PrivateKey(bio, key, 0, 0);
+	if (type == X509_ANY || type == X509_PEM) {
+		ok = !!PEM_read_bio_PrivateKey(bio, key, 0, "");
+	}
+
+	if (!ok && (type == X509_ANY || type == X509_DER)) {
+		ok = !!d2i_PrivateKey_bio(bio, key);
+	}
 
 	BIO_free(bio);
 
@@ -1027,7 +1101,7 @@ static int pk_toPEM(lua_State *L) {
 	bio = getbio(L);
 
 	for (i = 2; i <= top; i++) {
-		static const char *opts[] = {
+		static const char *const opts[] = {
 			"public", "PublicKey",
 			"private", "PrivateKey",
 //			"params", "Parameters",
@@ -1624,22 +1698,29 @@ int luaopen__openssl_x509_altname(lua_State *L) {
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 static int xc_new(lua_State *L) {
-	const char *pem;
+	const char *data;
 	size_t len;
 	X509 **ud;
 
-	lua_settop(L, 1);
+	lua_settop(L, 2);
 
 	ud = prepsimple(L, X509_CERT_CLASS);
 
-	if ((pem = luaL_optlstring(L, 1, NULL, &len))) {
+	if ((data = luaL_optlstring(L, 1, NULL, &len))) {
+		int type = optencoding(L, 2, "*", X509_ANY|X509_PEM|X509_DER);
 		BIO *tmp;
-		int ok;
+		int ok = 0;
 
-		if (!(tmp = BIO_new_mem_buf((char *)pem, len)))
+		if (!(tmp = BIO_new_mem_buf((char *)data, len)))
 			return throwssl(L, "x509.cert.new");
 
-		ok = !!PEM_read_bio_X509(tmp, ud, 0, ""); /* no password */
+		if (type == X509_PEM || type == X509_ANY) {
+			ok = !!(*ud = PEM_read_bio_X509(tmp, NULL, 0, "")); /* no password */
+		}
+
+		if (!ok && (type == X509_DER || type == X509_ANY)) {
+			ok = !!(*ud = d2i_X509_bio(tmp, NULL));
+		}
 
 		BIO_free(tmp);
 
@@ -2399,17 +2480,25 @@ static int xc_sign(lua_State *L) {
 
 static int xc__tostring(lua_State *L) {
 	X509 *crt = checksimple(L, 1, X509_CERT_CLASS);
-	int fmt = checkoption(L, 2, "pem", (const char *[]){ "pem", NULL });
+	int type = optencoding(L, 2, "pem", X509_PEM|X509_DER);
 	BIO *bio = getbio(L);
-	char *pem;
+	char *data;
 	long len;
 
-	if (!PEM_write_bio_X509(bio, crt))
-		return throwssl(L, "x509.cert:__tostring");
+	switch (type) {
+	case X509_PEM:
+		if (!PEM_write_bio_X509(bio, crt))
+			return throwssl(L, "x509.cert:__tostring");
+		break;
+	case X509_DER:
+		if (!i2d_X509_bio(bio, crt))
+			return throwssl(L, "x509.cert:__tostring");
+		break;
+	} /* switch() */
 
-	len = BIO_get_mem_data(bio, &pem);
+	len = BIO_get_mem_data(bio, &data);
 
-	lua_pushlstring(L, pem, len);
+	lua_pushlstring(L, data, len);
 
 	return 1;
 } /* xc__tostring() */
@@ -2486,26 +2575,34 @@ int luaopen__openssl_x509_cert(lua_State *L) {
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 static int xr_new(lua_State *L) {
-	const char *pem;
+	const char *data;
 	size_t len;
 	X509_REQ **ud;
 	X509 *crt;
 
 	lua_settop(L, 1);
+	lua_settop(L, 2);
 
 	ud = prepsimple(L, X509_CSR_CLASS);
 
 	if ((crt = testsimple(L, 1, X509_CERT_CLASS))) {
 		if (!(*ud = X509_to_X509_REQ(crt, 0, 0)))
 			return throwssl(L, "x509.csr.new");
-	} else if ((pem = luaL_optlstring(L, 1, NULL, &len))) {
+	} else if ((data = luaL_optlstring(L, 1, NULL, &len))) {
+		int type = optencoding(L, 2, "*", X509_ANY|X509_PEM|X509_DER);
 		BIO *tmp;
-		int ok;
+		int ok = 0;
 
-		if (!(tmp = BIO_new_mem_buf((char *)pem, len)))
+		if (!(tmp = BIO_new_mem_buf((char *)data, len)))
 			return throwssl(L, "x509.csr.new");
 
-		ok = !!PEM_read_bio_X509_REQ(tmp, ud, 0, ""); /* no password */
+		if (type == X509_PEM || type == X509_ANY) {
+			ok = !!(*ud = PEM_read_bio_X509_REQ(tmp, NULL, 0, "")); /* no password */
+		}
+
+		if (!ok && (type == X509_DER || type == X509_ANY)) {
+			ok = !!(*ud = d2i_X509_REQ_bio(tmp, NULL));
+		}
 
 		BIO_free(tmp);
 
@@ -2612,17 +2709,25 @@ static int xr_sign(lua_State *L) {
 
 static int xr__tostring(lua_State *L) {
 	X509_REQ *csr = checksimple(L, 1, X509_CSR_CLASS);
-	int fmt = checkoption(L, 2, "pem", (const char *[]){ "pem", NULL });
+	int type = optencoding(L, 2, "pem", X509_PEM|X509_DER);
 	BIO *bio = getbio(L);
-	char *pem;
+	char *data;
 	long len;
 
-	if (!PEM_write_bio_X509_REQ(bio, csr))
-		return throwssl(L, "x509.csr:__tostring");
+	switch (type) {
+	case X509_PEM:
+		if (!PEM_write_bio_X509_REQ(bio, csr))
+			return throwssl(L, "x509.csr:__tostring");
+		break;
+	case X509_DER:
+		if (!i2d_X509_REQ_bio(bio, csr))
+			return throwssl(L, "x509.csr:__tostring");
+		break;
+	} /* switch() */
 
-	len = BIO_get_mem_data(bio, &pem);
+	len = BIO_get_mem_data(bio, &data);
 
-	lua_pushlstring(L, pem, len);
+	lua_pushlstring(L, data, len);
 
 	return 1;
 } /* xr__tostring() */
@@ -3063,7 +3168,7 @@ int luaopen__openssl_x509_store_context(lua_State *L) {
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 static int sx_new(lua_State *L) {
-	static const char *opts[] = {
+	static const char *const opts[] = {
 		"SSLv2", "SSLv3", "SSLv23", "SSL", "TLSv1", "TLS", NULL
 	};
 	/* later versions of SSL declare a const qualifier on the return type */