Merge branch 'daurnimator-58-pkey-encrypt+decrypt'

1 files changed, 142 insertions, 0 deletions

diff --git a/src/openssl.c b/src/openssl.c
index c33d934..38c9888 100644
--- a/src/openssl.c
+++ b/src/openssl.c
@@ -193,6 +193,10 @@
 #define HAVE_EVP_PKEY_BASE_ID OPENSSL_PREREQ(1,1,0)
 #endif
 
+#ifndef HAVE_EVP_PKEY_CTX_NEW
+#define HAVE_EVP_PKEY_CTX_NEW (OPENSSL_PREREQ(1,0,0) || LIBRESSL_PREREQ(2,0,0))
+#endif
+
 #ifndef HAVE_EVP_PKEY_GET0
 #define HAVE_EVP_PKEY_GET0 OPENSSL_PREREQ(1,1,0)
 #endif
@@ -225,6 +229,10 @@
 #define HAVE_RSA_GET0_KEY OPENSSL_PREREQ(1,1,0)
 #endif
 
+#ifndef HAVE_RSA_PKCS1_PSS_PADDING
+#define HAVE_RSA_PKCS1_PSS_PADDING (defined RSA_PKCS1_PSS_PADDING || OPENSSL_PREREQ(1,0,0) || LIBRESSL_PREREQ(2,0,0))
+#endif
+
 #ifndef HAVE_RSA_SET0_CRT_PARAMS
 #define HAVE_RSA_SET0_CRT_PARAMS OPENSSL_PREREQ(1,1,0)
 #endif
@@ -3281,6 +3289,123 @@ static int pk_setPrivateKey(lua_State *L) {
 	return 1;
 } /* pk_setPrivateKey() */
 
+#if HAVE_EVP_PKEY_CTX_NEW
+static int pk_decrypt(lua_State *L) {
+	size_t outlen, inlen;
+	EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
+	EVP_PKEY_CTX *ctx;
+	const char *str = luaL_checklstring(L, 2, &inlen);
+	BIO *bio;
+	BUF_MEM *buf;
+	int rsaPadding = RSA_PKCS1_PADDING; /* default for `openssl rsautl` */
+	int base_type = EVP_PKEY_base_id(key);
+
+	if (lua_istable(L, 3)) {
+		if (base_type == EVP_PKEY_RSA) {
+			lua_getfield(L, 3, "rsaPadding");
+			rsaPadding = luaL_optint(L, -1, rsaPadding);
+			lua_pop(L, 1);
+		}
+	}
+
+	bio = getbio(L);
+	BIO_get_mem_ptr(bio, &buf);
+
+	if (!(ctx = EVP_PKEY_CTX_new(key, NULL)))
+		goto sslerr;
+
+	if (EVP_PKEY_decrypt_init(ctx) <= 0)
+		goto sslerr;
+
+	if (base_type == EVP_PKEY_RSA && !EVP_PKEY_CTX_set_rsa_padding(ctx, rsaPadding))
+		goto sslerr;
+
+	if (EVP_PKEY_decrypt(ctx, NULL, &outlen, (const unsigned char *)str, inlen) <= 0)
+		goto sslerr;
+
+	if (!BUF_MEM_grow_clean(buf, outlen))
+		goto sslerr;
+
+	if (EVP_PKEY_decrypt(ctx, (unsigned char *)buf->data, &outlen, (const unsigned char *)str, inlen) <= 0)
+		goto sslerr;
+
+	EVP_PKEY_CTX_free(ctx);
+	ctx = NULL;
+
+	lua_pushlstring(L, buf->data, outlen);
+
+	BIO_reset(bio);
+
+	return 1;
+sslerr:
+	if (ctx) {
+		EVP_PKEY_CTX_free(ctx);
+		ctx = NULL;
+	}
+	BIO_reset(bio);
+
+	return auxL_error(L, auxL_EOPENSSL, "pkey:decrypt");
+} /* pk_decrypt() */
+#endif
+
+#if HAVE_EVP_PKEY_CTX_NEW
+static int pk_encrypt(lua_State *L) {
+	size_t outlen, inlen;
+	EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
+	EVP_PKEY_CTX *ctx;
+	const char *str = luaL_checklstring(L, 2, &inlen);
+	BIO *bio;
+	BUF_MEM *buf;
+	int rsaPadding = RSA_PKCS1_PADDING; /* default for `openssl rsautl` */
+	int base_type = EVP_PKEY_base_id(key);
+
+	if (lua_istable(L, 3)) {
+		if (base_type == EVP_PKEY_RSA) {
+			lua_getfield(L, 3, "rsaPadding");
+			rsaPadding = luaL_optint(L, -1, rsaPadding);
+			lua_pop(L, 1);
+		}
+	}
+
+	bio = getbio(L);
+	BIO_get_mem_ptr(bio, &buf);
+
+	if (!(ctx = EVP_PKEY_CTX_new(key, NULL)))
+		goto sslerr;
+
+	if (EVP_PKEY_encrypt_init(ctx) <= 0)
+		goto sslerr;
+
+	if (base_type == EVP_PKEY_RSA && !EVP_PKEY_CTX_set_rsa_padding(ctx, rsaPadding))
+		goto sslerr;
+
+	if (EVP_PKEY_encrypt(ctx, NULL, &outlen, (const unsigned char *)str, inlen) <= 0)
+		goto sslerr;
+
+	if (!BUF_MEM_grow_clean(buf, outlen))
+		goto sslerr;
+
+	if (EVP_PKEY_encrypt(ctx, (unsigned char *)buf->data, &outlen, (const unsigned char *)str, inlen) <= 0)
+		goto sslerr;
+
+	EVP_PKEY_CTX_free(ctx);
+	ctx = NULL;
+
+	lua_pushlstring(L, buf->data, outlen);
+
+	BIO_reset(bio);
+
+	return 1;
+sslerr:
+	if (ctx) {
+		EVP_PKEY_CTX_free(ctx);
+		ctx = NULL;
+	}
+	BIO_reset(bio);
+
+	return auxL_error(L, auxL_EOPENSSL, "pkey:encrypt");
+} /* pk_encrypt() */
+#endif
 
 static int pk_sign(lua_State *L) {
 	EVP_PKEY *key = checksimple(L, 1, PKEY_CLASS);
@@ -4001,6 +4126,10 @@ static const auxL_Reg pk_methods[] = {
 	{ "type",          &pk_type },
 	{ "setPublicKey",  &pk_setPublicKey },
 	{ "setPrivateKey", &pk_setPrivateKey },
+#if HAVE_EVP_PKEY_CTX_NEW
+	{ "decrypt",       &pk_decrypt },
+	{ "encrypt",       &pk_encrypt },
+#endif
 	{ "sign",          &pk_sign },
 	{ "verify",        &pk_verify },
 	{ "getDefaultDigestName", &pk_getDefaultDigestName },
@@ -4039,10 +4168,23 @@ static void pk_luainit(lua_State *L, _Bool reset) {
 	lua_pop(L, 2);
 } /* pk_luainit() */
 
+static const auxL_IntegerReg pk_rsa_pad_opts[] = {
+	{ "RSA_PKCS1_PADDING", RSA_PKCS1_PADDING }, // PKCS#1 padding
+	{ "RSA_SSLV23_PADDING", RSA_SSLV23_PADDING }, // SSLv23 padding
+	{ "RSA_NO_PADDING", RSA_NO_PADDING }, // no padding
+	{ "RSA_PKCS1_OAEP_PADDING", RSA_PKCS1_OAEP_PADDING }, // OAEP padding (encrypt and decrypt only)
+	{ "RSA_X931_PADDING", RSA_X931_PADDING }, // (signature operations only)
+#if HAVE_RSA_PKCS1_PSS_PADDING
+	{ "RSA_PKCS1_PSS_PADDING", RSA_PKCS1_PSS_PADDING }, // (sign and verify only)
+#endif
+	{ NULL, 0 },
+};
+
 int luaopen__openssl_pkey(lua_State *L) {
 	initall(L);
 
 	auxL_newlib(L, pk_globals, 0);
+	auxL_setintegers(L, pk_rsa_pad_opts);
 
 	return 1;
 } /* luaopen__openssl_pkey() */