Patch to add PKCS#8 key generation to lua-resty-rsa

rsa.lua.patch

--- rsa.lua.oem	2017-11-13 01:02:14.272018211 -0500
+++ rsa.lua	2017-11-13 18:54:02.587237476 -0500
@@ -29,13 +29,18 @@
     PKCS8 = "PKCS#8",
 }
 
+--[[ tarkin000: MOD added -
+ int PEM_write_bio_RSA_PUBKEY(BIO *bp, RSA *x);
+ int PEM_write_bio_PKCS8PrivateKey(BIO *bp, EVP_PKEY *x, const EVP_CIPHER *enc, char *kstr, int klen, pem_password_cb *cb, void *u);
+ int EVP_PKEY_set1_RSA(EVP_PKEY *e, RSA *r);
+]]--
 
 ffi.cdef[[
 typedef struct bio_st BIO;
 typedef struct bio_method_st BIO_METHOD;
 BIO_METHOD *BIO_s_mem(void);
 BIO * BIO_new(BIO_METHOD *type);
-int	BIO_puts(BIO *bp,const char *buf);
+int BIO_puts(BIO *bp,const char *buf);
 void BIO_vfree(BIO *a);
 
 typedef struct rsa_st RSA;
@@ -43,7 +48,7 @@
 void RSA_free(RSA *rsa);
 typedef int pem_password_cb(char *buf, int size, int rwflag, void *userdata);
 RSA * PEM_read_bio_RSAPrivateKey(BIO *bp, RSA **rsa, pem_password_cb *cb,
-								void *u);
+                                 void *u);
 RSA * PEM_read_bio_RSAPublicKey(BIO *bp, RSA **rsa, pem_password_cb *cb,
                                 void *u);
 RSA * PEM_read_bio_RSA_PUBKEY(BIO *bp, RSA **rsa, pem_password_cb *cb,
@@ -65,6 +70,7 @@
                                 unsigned char *kstr, int klen,
                                 pem_password_cb *cb, void *u);
 int PEM_write_bio_RSAPublicKey(BIO *bp, RSA *x);
+int PEM_write_bio_RSA_PUBKEY(BIO *bp, RSA *x);
 
 long BIO_ctrl(BIO *bp, int cmd, long larg, void *parg);
 int BIO_read(BIO *b, void *data, int len);
@@ -75,7 +81,6 @@
 
 EVP_PKEY *EVP_PKEY_new(void);
 void EVP_PKEY_free(EVP_PKEY *key);
-int EVP_PKEY_set1_RSA(EVP_PKEY *pkey,RSA *key);
 
 EVP_PKEY_CTX *EVP_PKEY_CTX_new(EVP_PKEY *pkey, ENGINE *e);
 void EVP_PKEY_CTX_free(EVP_PKEY_CTX *ctx);
@@ -95,6 +100,9 @@
                      unsigned char *out, size_t *outlen,
                      const unsigned char *in, size_t inlen);
 
+int EVP_PKEY_set1_RSA(EVP_PKEY *pkey, RSA *key);
+int PEM_write_bio_PKCS8PrivateKey(BIO *bp, EVP_PKEY *x, const EVP_CIPHER *enc, char *kstr, int klen, pem_password_cb *cb, void *u);
+
 void OpenSSL_add_all_digests(void);
 typedef struct env_md_st EVP_MD;
 typedef struct env_md_ctx_st EVP_MD_CTX;
@@ -106,6 +114,7 @@
 int EVP_DigestUpdate(EVP_MD_CTX *ctx, const unsigned char *in, int inl);
 int EVP_SignFinal(EVP_MD_CTX *ctx,unsigned char *sig,unsigned int *s, EVP_PKEY *pkey);
 int EVP_VerifyFinal(EVP_MD_CTX *ctx,unsigned char *sigbuf, unsigned int siglen,EVP_PKEY *pkey);
+int EVP_PKEY_set1_RSA(EVP_PKEY *e, RSA *r);
 ]]
 --[[
 # define EVP_PKEY_CTX_set_rsa_padding(ctx, pad) \
@@ -140,7 +149,8 @@
 end
 
 -- Follow the calling style to avoid careless mistake.
-function _M.generate_rsa_keys(_, bits)
+-- tarkin000: MOD - option to generate PKCS#8 public key
+function _M.generate_rsa_keys(_, bits, pkcs8)
     local rsa = C.RSA_new()
     ffi_gc(rsa, C.RSA_free)
     local bn = C.BN_new()
@@ -158,8 +168,14 @@
 
     local pub_key_bio = C.BIO_new(C.BIO_s_mem())
     ffi_gc(pub_key_bio, C.BIO_vfree)
-    if C.PEM_write_bio_RSAPublicKey(pub_key_bio, rsa) ~= 1 then
-        return nil, ssl_err()
+    if pkcs8 == true then 
+        if C.PEM_write_bio_RSA_PUBKEY(pub_key_bio, rsa) ~= 1 then
+            return nil, ssl_err()
+        end
+    else
+        if C.PEM_write_bio_RSAPublicKey(pub_key_bio, rsa) ~= 1 then
+             return nil, ssl_err()
+        end
     end
     local public_key, err = read_bio(pub_key_bio)
     if not public_key then
@@ -168,8 +184,19 @@
 
     local priv_key_bio = C.BIO_new(C.BIO_s_mem())
     ffi_gc(priv_key_bio, C.BIO_vfree)
-    if C.PEM_write_bio_RSAPrivateKey(priv_key_bio, rsa, nil, nil, 0, nil, nil) ~= 1 then
-        return nil, ssl_err()
+    if pkcs8 == true then
+        local pk = C.EVP_PKEY_new()
+        ffi_gc(pk, C.EVP_PKEY_free)
+        if C.EVP_PKEY_set1_RSA(pk,rsa) ~= 1 then
+            return nil, ssl_err()
+        end
+        if C.PEM_write_bio_PKCS8PrivateKey(priv_key_bio, pk, nil, nil, 0, nil, nil) ~= 1 then
+            return nil, ssl_err()
+        end
+    else
+        if C.PEM_write_bio_RSAPrivateKey(priv_key_bio, rsa, nil, nil, 0, nil, nil) ~= 1 then
+            return nil, ssl_err()
+        end
     end
     local private_key
     private_key, err = read_bio(priv_key_bio)