From ace7fbd0bda55e1ea4343abe2fe51619fc34a40b Mon Sep 17 00:00:00 2001
From: dmiller <dmiller@e0a8ed71-7df4-0310-8962-fdc924857419>
Date: Thu, 20 Jun 2019 23:11:10 +0000
Subject: [PATCH] Fix a memory leak when getting SSL cert with RSA key in NSE

---
 nse_openssl.cc  |  9 +++++++--
 nse_ssl_cert.cc | 53 +++++++++++++++++++++++++++----------------------
 2 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/nse_openssl.cc b/nse_openssl.cc
index 55c2ffe82..95b9a8bc9 100644
--- a/nse_openssl.cc
+++ b/nse_openssl.cc
@@ -46,14 +46,17 @@
 
 typedef struct bignum_data {
   BIGNUM * bn;
+  bool should_free;
 } bignum_data_t;
 
-static int nse_pushbn( lua_State *L, BIGNUM *num )
+static int nse_pushbn( lua_State *L, BIGNUM *num)
 {
   bignum_data_t * data = (bignum_data_t *) lua_newuserdata( L, sizeof(bignum_data_t));
   luaL_getmetatable( L, "BIGNUM" );
   lua_setmetatable( L, -2 );
   data->bn = num;
+  /* Currently this is true for all uses in this file. */
+  data->should_free = true;
   return 1;
 }
 
@@ -235,7 +238,9 @@ static int l_bignum_bn2hex( lua_State *L ) /** bignum_bn2hex( BIGNUM bn ) */
 static int l_bignum_free( lua_State *L ) /** bignum_free( bignum ) */
 {
   bignum_data_t * userdata = (bignum_data_t *) luaL_checkudata(L, 1, "BIGNUM");
-  BN_clear_free( userdata->bn );
+  if (userdata->should_free) {
+    BN_clear_free( userdata->bn );
+  }
   return 0;
 }
 
diff --git a/nse_ssl_cert.cc b/nse_ssl_cert.cc
index 8b26aa12c..3bd509ddd 100644
--- a/nse_ssl_cert.cc
+++ b/nse_ssl_cert.cc
@@ -170,12 +170,11 @@ struct cert_userdata {
   int attributes_table;
 };
 
-/* from nse_openssl.cc */
 typedef struct bignum_data {
   BIGNUM * bn;
+  bool should_free;
 } bignum_data_t;
 
-
 SSL *nse_nsock_get_ssl(lua_State *L);
 
 /* This is a reference to a table that will be used as the metatable for
@@ -633,6 +632,7 @@ static int parse_ssl_cert(lua_State *L, X509 *cert)
   if (pubkey == NULL) {
     lua_pushnil(L);
     lua_pushfstring(L, "Error parsing cert: %s", ERR_error_string(ERR_get_error(), NULL));
+    X509_free(cert);
     return 2;
   }
 #define NSE_NUM_PKEY_FIELDS 4
@@ -651,28 +651,33 @@ static int parse_ssl_cert(lua_State *L, X509 *cert)
 #endif
   if (pkey_type == EVP_PKEY_RSA) {
     RSA *rsa = EVP_PKEY_get1_RSA(pubkey);
-    /* exponent */
-    bignum_data_t * data = (bignum_data_t *) lua_newuserdata( L, sizeof(bignum_data_t));
-    luaL_getmetatable( L, "BIGNUM" );
-    lua_setmetatable( L, -2 );
-  #if HAVE_OPAQUE_STRUCTS
-    const BIGNUM *n, *e;
-    RSA_get0_key(rsa, &n, &e, NULL);
-    data->bn = (BIGNUM*) e;
-  #else
-    data->bn = rsa->e;
-  #endif
-    lua_setfield(L, -2, "exponent");
-    /* modulus */
-    data = (bignum_data_t *) lua_newuserdata( L, sizeof(bignum_data_t));
-    luaL_getmetatable( L, "BIGNUM" );
-    lua_setmetatable( L, -2 );
-  #if HAVE_OPAQUE_STRUCTS
-    data->bn = (BIGNUM*) n;
-  #else
-    data->bn = rsa->n;
-  #endif
-    lua_setfield(L, -2, "modulus");
+    if (rsa) {
+      /* exponent */
+      bignum_data_t * data = (bignum_data_t *) lua_newuserdata( L, sizeof(bignum_data_t));
+      luaL_getmetatable( L, "BIGNUM" );
+      lua_setmetatable( L, -2 );
+      data->should_free = false;
+#if HAVE_OPAQUE_STRUCTS
+      const BIGNUM *n, *e;
+      RSA_get0_key(rsa, &n, &e, NULL);
+      data->bn = (BIGNUM*) e;
+#else
+      data->bn = rsa->e;
+#endif
+      lua_setfield(L, -2, "exponent");
+      /* modulus */
+      data = (bignum_data_t *) lua_newuserdata( L, sizeof(bignum_data_t));
+      luaL_getmetatable( L, "BIGNUM" );
+      lua_setmetatable( L, -2 );
+      data->should_free = false;
+#if HAVE_OPAQUE_STRUCTS
+      data->bn = (BIGNUM*) n;
+#else
+      data->bn = rsa->n;
+#endif
+      lua_setfield(L, -2, "modulus");
+      RSA_free(rsa);
+    }
   }
   lua_pushstring(L, pkey_type_to_string(pkey_type));
   lua_setfield(L, -2, "type");