Fix build with OpenSSL 3.0

Updated all OpenSSL code to OpenSSL 3.0 API with no deprecated functions. Some
NSE functions were changed:
* openssl.rc4_options and openssl.rc4 were removed in favor of openssl.encrypt
* openssl.bignum_pseudo_rand is now an alias for openssl.bignum_rand
* openssl.bignum_is_prime and openssl.bignum_is_safe_prime will now ignore the
  nchecks parameter, using a secure default instead.

ncat/http_digest.c

@@ -77,7 +77,7 @@
 #include "ncat.h"
 #include "http.h"
 
-#include <openssl/md5.h>
+#include <openssl/evp.h>
 #include <openssl/rand.h>
 
 /* What's a good length for this? I think it exists only to prevent us from
@@ -133,14 +133,19 @@ int http_digest_init_secret(void)
     return 0;
 }
 
+#if OPENSSL_API_LEVEL < 10100
+#define EVP_MD_CTX_new EVP_MD_CTX_create
+#define EVP_MD_CTX_free EVP_MD_CTX_destroy
+#endif
 static char *make_nonce(const struct timeval *tv)
 {
     char *buf = NULL;
     size_t size = 0, offset = 0;
-    MD5_CTX md5;
+    EVP_MD_CTX *md5;
-    unsigned char hashbuf[MD5_DIGEST_LENGTH];
+    unsigned char hashbuf[EVP_MAX_MD_SIZE];
-    char hash_hex[MD5_DIGEST_LENGTH * 2 + 1];
+    char hash_hex[EVP_MAX_MD_SIZE * 2 + 1];
     char time_buf[32];
+    unsigned int hash_size = 0;
 
     /* Crash if someone forgot to call http_digest_init_secret. */
     if (!secret_initialized)
@@ -148,13 +153,13 @@ static char *make_nonce(const struct timeval *tv)
 
     Snprintf(time_buf, sizeof(time_buf), "%lu.%06lu",
         (long unsigned) tv->tv_sec, (long unsigned) tv->tv_usec);
-
+    md5 = EVP_MD_CTX_new();
-    MD5_Init(&md5);
+    EVP_DigestInit_ex(md5, EVP_md5(), NULL);
-    MD5_Update(&md5, secret, sizeof(secret));
+    EVP_DigestUpdate(md5, secret, sizeof(secret));
-    MD5_Update(&md5, ":", 1);
+    EVP_DigestUpdate(md5, ":", 1);
-    MD5_Update(&md5, time_buf, strlen(time_buf));
+    EVP_DigestUpdate(md5, time_buf, strlen(time_buf));
-    MD5_Final(hashbuf, &md5);
+    EVP_DigestFinal_ex(md5, hashbuf, &hash_size);
-    enhex(hash_hex, hashbuf, sizeof(hashbuf));
+    enhex(hash_hex, hashbuf, hash_size);
 
     strbuf_sprintf(&buf, &size, &offset, "%s-%s", time_buf, hash_hex);
 
@@ -163,51 +168,54 @@ static char *make_nonce(const struct timeval *tv)
 
 /* Arguments are assumed to be non-NULL, with the exception of nc and cnonce,
    which may be garbage only if qop == QOP_NONE. */
-static void make_response(char buf[MD5_DIGEST_LENGTH * 2 + 1],
+static void make_response(char buf[EVP_MAX_MD_SIZE * 2 + 1],
     const char *username, const char *realm, const char *password,
     const char *method, const char *uri, const char *nonce,
     enum http_digest_qop qop, const char *nc, const char *cnonce)
 {
-    char HA1_hex[MD5_DIGEST_LENGTH * 2 + 1], HA2_hex[MD5_DIGEST_LENGTH * 2 + 1];
+    char HA1_hex[EVP_MAX_MD_SIZE * 2 + 1], HA2_hex[EVP_MAX_MD_SIZE * 2 + 1];
-    unsigned char hashbuf[MD5_DIGEST_LENGTH];
+    unsigned char hashbuf[EVP_MAX_MD_SIZE];
-    MD5_CTX md5;
+    EVP_MD_CTX *md5;
+    unsigned int hash_size = 0;
+    const EVP_MD *md = EVP_md5();
 
     /* Calculate H(A1). */
-    MD5_Init(&md5);
+    md5 = EVP_MD_CTX_new();
-    MD5_Update(&md5, username, strlen(username));
+    EVP_DigestInit_ex(md5, md, NULL);
-    MD5_Update(&md5, ":", 1);
+    EVP_DigestUpdate(md5, username, strlen(username));
-    MD5_Update(&md5, realm, strlen(realm));
+    EVP_DigestUpdate(md5, ":", 1);
-    MD5_Update(&md5, ":", 1);
+    EVP_DigestUpdate(md5, realm, strlen(realm));
-    MD5_Update(&md5, password, strlen(password));
+    EVP_DigestUpdate(md5, ":", 1);
-    MD5_Final(hashbuf, &md5);
+    EVP_DigestUpdate(md5, password, strlen(password));
-    enhex(HA1_hex, hashbuf, sizeof(hashbuf));
+    EVP_DigestFinal_ex(md5, hashbuf, &hash_size);
+    enhex(HA1_hex, hashbuf, hash_size);
 
     /* Calculate H(A2). */
-    MD5_Init(&md5);
+    EVP_DigestInit_ex(md5, md, NULL);
-    MD5_Update(&md5, method, strlen(method));
+    EVP_DigestUpdate(md5, method, strlen(method));
-    MD5_Update(&md5, ":", 1);
+    EVP_DigestUpdate(md5, ":", 1);
-    MD5_Update(&md5, uri, strlen(uri));
+    EVP_DigestUpdate(md5, uri, strlen(uri));
-    MD5_Final(hashbuf, &md5);
+    EVP_DigestFinal_ex(md5, hashbuf, &hash_size);
-    enhex(HA2_hex, hashbuf, sizeof(hashbuf));
+    enhex(HA2_hex, hashbuf, hash_size);
 
     /* Calculate response. */
-    MD5_Init(&md5);
+    EVP_DigestInit_ex(md5, md, NULL);
-    MD5_Update(&md5, HA1_hex, strlen(HA1_hex));
+    EVP_DigestUpdate(md5, HA1_hex, strlen(HA1_hex));
-    MD5_Update(&md5, ":", 1);
+    EVP_DigestUpdate(md5, ":", 1);
-    MD5_Update(&md5, nonce, strlen(nonce));
+    EVP_DigestUpdate(md5, nonce, strlen(nonce));
     if (qop == QOP_AUTH) {
-        MD5_Update(&md5, ":", 1);
+        EVP_DigestUpdate(md5, ":", 1);
-        MD5_Update(&md5, nc, strlen(nc));
+        EVP_DigestUpdate(md5, nc, strlen(nc));
-        MD5_Update(&md5, ":", 1);
+        EVP_DigestUpdate(md5, ":", 1);
-        MD5_Update(&md5, cnonce, strlen(cnonce));
+        EVP_DigestUpdate(md5, cnonce, strlen(cnonce));
-        MD5_Update(&md5, ":", 1);
+        EVP_DigestUpdate(md5, ":", 1);
-        MD5_Update(&md5, "auth", strlen("auth"));
+        EVP_DigestUpdate(md5, "auth", strlen("auth"));
     }
-    MD5_Update(&md5, ":", 1);
+    EVP_DigestUpdate(md5, ":", 1);
-    MD5_Update(&md5, HA2_hex, strlen(HA2_hex));
+    EVP_DigestUpdate(md5, HA2_hex, strlen(HA2_hex));
-    MD5_Final(hashbuf, &md5);
+    EVP_DigestFinal_ex(md5, hashbuf, &hash_size);
 
-    enhex(buf, hashbuf, sizeof(hashbuf));
+    enhex(buf, hashbuf, hash_size);
 }
 
 /* Extract the issuance time from a nonce (without checking other aspects of
@@ -258,7 +266,7 @@ char *http_digest_proxy_authorization(const struct http_challenge *challenge,
        varying client nonce count. */
     static const u32 nc = 0x00000001;
 
-    char response_hex[MD5_DIGEST_LENGTH * 2 + 1];
+    char response_hex[EVP_MAX_MD_SIZE * 2 + 1];
     unsigned char cnonce[CNONCE_LENGTH];
     char cnonce_buf[CNONCE_LENGTH * 2 + 1];
     char nc_buf[8 + 1];
@@ -322,7 +330,7 @@ int http_digest_check_credentials(const char *username, const char *realm,
     const char *password, const char *method,
     const struct http_credentials *credentials)
 {
-    char response_hex[MD5_DIGEST_LENGTH * 2 + 1];
+    char response_hex[EVP_MAX_MD_SIZE * 2 + 1];
     struct timeval tv;
     char *nonce;
 

ncat/ncat.h

@@ -66,10 +66,6 @@
 
 #include <nbase.h>
 
-#ifdef HAVE_OPENSSL
-#include <openssl/ssl.h>
-#endif
-
 #include "nsock.h"
 #include "util.h"
 #include "sys_wrap.h"

ncat/ncat_connect.c

@@ -80,6 +80,11 @@
 #ifdef HAVE_OPENSSL
 #include <openssl/ssl.h>
 #include <openssl/err.h>
+
+/* Deprecated in OpenSSL 3.0 */
+#if OPENSSL_API_LEVEL >= 30000
+#define SSL_get_peer_certificate SSL_get1_peer_certificate
+#endif
 #endif
 
 #ifdef WIN32

ncat/ncat_ssl.c

@@ -67,7 +67,6 @@
 
 #include <stdio.h>
 #include <openssl/ssl.h>
-#include <openssl/bn.h>
 #include <openssl/err.h>
 #include <openssl/rsa.h>
 #include <openssl/rand.h>
@@ -81,6 +80,14 @@
 #define FUNC_ASN1_STRING_data ASN1_STRING_data
 #endif
 
+#if OPENSSL_API_LEVEL >= 30000
+#include <openssl/provider.h>
+/* Deprecated in OpenSSL 3.0 */
+#define SSL_get_peer_certificate SSL_get1_peer_certificate
+#else
+#include <openssl/bn.h>
+#endif
+
 /* Required for windows compilation to Eliminate APPLINK errors.
    See http://www.openssl.org/support/faq.html#PROG2 */
 #ifdef WIN32
@@ -110,6 +117,15 @@ SSL_CTX *setup_ssl_listen(void)
     OpenSSL_add_all_algorithms();
     ERR_load_crypto_strings();
     SSL_load_error_strings();
+#elif OPENSSL_API_LEVEL >= 30000
+  if (NULL == OSSL_PROVIDER_load(NULL, "legacy"))
+  {
+    loguser("OpenSSL legacy provider failed to load.\n");
+  }
+  if (NULL == OSSL_PROVIDER_load(NULL, "default"))
+  {
+    loguser("OpenSSL default provider failed to load.\n");
+  }
 #endif
 
     /* RAND_status initializes the random number generator through a variety of
@@ -455,14 +471,16 @@ int ssl_post_connect_check(SSL *ssl, const char *hostname)
    "Making Certificates"; and apps/req.c in the OpenSSL source. */
 static int ssl_gen_cert(X509 **cert, EVP_PKEY **key)
 {
-    RSA *rsa = NULL;
     X509_NAME *subj;
     X509_EXTENSION *ext;
     X509V3_CTX ctx;
-    BIGNUM *bne = NULL;
     const char *commonName = "localhost";
     char dNSName[128];
-    int rc, ret=0;
+    int rc;
+#if OPENSSL_API_LEVEL < 30000
+    int ret = 0;
+    RSA *rsa = NULL;
+    BIGNUM *bne = NULL;
 
     *cert = NULL;
     *key = NULL;
@@ -491,6 +509,12 @@ static int ssl_gen_cert(X509 **cert, EVP_PKEY **key)
         RSA_free(rsa);
         goto err;
     }
+#else
+    *cert = NULL;
+    *key = EVP_RSA_gen(DEFAULT_KEY_BITS);
+    if (*key == NULL)
+        goto err;
+#endif
 
     /* Generate a certificate. */
     *cert = X509_new();

ncat/ncat_ssl.h

@@ -67,6 +67,18 @@
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 
+/* OPENSSL_API_LEVEL per OpenSSL 3.0: decimal MMmmpp */
+#ifndef OPENSSL_API_LEVEL
+# if OPENSSL_API_COMPAT < 0x900000L
+#  define OPENSSL_API_LEVEL (OPENSSL_API_COMPAT)
+# else
+#  define OPENSSL_API_LEVEL \
+     (((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000  \
+      + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+      + ((OPENSSL_API_COMPAT >> 12) & 0xFF))
+# endif
+#endif
+
 #define NCAT_CA_CERTS_FILE "ca-bundle.crt"
 
 enum {

ncat/test/test-wildcard.c

@@ -12,7 +12,6 @@ are rejected. The SSL transactions happen over OpenSSL BIO pairs.
 #include <unistd.h>
 
 #include <openssl/bio.h>
-#include <openssl/bn.h>
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 #include <openssl/rsa.h>
@@ -20,6 +19,10 @@ are rejected. The SSL transactions happen over OpenSSL BIO pairs.
 #include <openssl/x509v3.h>
 
 #include "ncat_core.h"
+#include "ncat_ssl.h"
+#if OPENSSL_API_LEVEL < 30000
+#include <openssl/bn.h>
+#endif
 
 #define KEY_BITS 2048
 
@@ -291,9 +294,10 @@ stack_err:
 static int gen_cert(X509 **cert, EVP_PKEY **key,
     const struct lstr commonNames[], const struct lstr dNSNames[])
 {
+#if OPENSSL_API_LEVEL < 30000
+    int rc, ret=0;
     RSA *rsa = NULL;
     BIGNUM *bne = NULL;
-    int rc, ret=0;
 
     *cert = NULL;
     *key = NULL;
@@ -322,6 +326,12 @@ static int gen_cert(X509 **cert, EVP_PKEY **key,
         RSA_free(rsa);
         goto err;
     }
+#else
+    *cert = NULL;
+    *key = EVP_RSA_gen(KEY_BITS);
+    if (*key == NULL)
+        goto err;
+#endif
 
     /* Generate a certificate. */
     *cert = X509_new();

nse_openssl.cc

@@ -8,18 +8,10 @@
 
 #include <openssl/bn.h>
 #include <openssl/crypto.h>
-#include <openssl/des.h>
 #include <openssl/err.h>
 #include <openssl/evp.h>
 #include <openssl/hmac.h>
-#include <openssl/md4.h>
-#include <openssl/md5.h>
 #include <openssl/rand.h>
-#ifndef OPENSSL_NO_RC4
-#include <openssl/rc4.h>
-#endif
-#include <openssl/ripemd.h>
-#include <openssl/sha.h>
 
 #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined LIBRESSL_VERSION_NUMBER
 #define HAVE_OPAQUE_STRUCTS 1
@@ -37,13 +29,34 @@
 #endif
 
 #include "nse_lua.h"
-#if OPENSSL_API_COMPAT >= 0x10100000L
+/* Needed for get_random_bytes */
-  /* Needed for get_random_bytes, since RAND_pseudo_bytes is gone */
+#include <nbase.h>
-  #include <nbase.h>
+#include "NmapOps.h"
-#endif
+#include "output.h"
+extern NmapOps o;
 
 #include "nse_openssl.h"
 
+/* OPENSSL_API_LEVEL per OpenSSL 3.0: decimal MMmmpp */
+#ifndef OPENSSL_API_LEVEL
+# if OPENSSL_API_COMPAT < 0x900000L
+#  define OPENSSL_API_LEVEL (OPENSSL_API_COMPAT)
+# else
+#  define OPENSSL_API_LEVEL \
+     (((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000  \
+      + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+      + ((OPENSSL_API_COMPAT >> 12) & 0xFF))
+# endif
+#endif
+
+
+#if OPENSSL_API_LEVEL >= 30000
+#include <openssl/provider.h>
+#endif
+
+#define NSE_SSL_LUA_ERR(_L) \
+    luaL_error(_L, "OpenSSL error: %s", ERR_error_string(ERR_get_error(), NULL))
+
 typedef struct bignum_data {
   BIGNUM * bn;
   bool should_free;
@@ -93,14 +106,6 @@ static int l_bignum_rand( lua_State *L ) /** bignum_rand( number bits ) */
   return nse_pushbn(L, num);
 }
 
-static int l_bignum_pseudo_rand( lua_State *L ) /** bignum_pseudo_rand( number bits ) */
-{
-  size_t bits = luaL_checkinteger( L, 1 );
-  BIGNUM * num = BN_new();
-  BN_pseudo_rand( num, bits, -1, 0 );
-  return nse_pushbn(L, num);
-}
-
 static int l_bignum_mod_exp( lua_State *L ) /** bignum_mod_exp( BIGNUM a, BIGNUM p, BIGNUM m ) */
 {
   bignum_data_t * a = (bignum_data_t *) luaL_checkudata(L, 1, "BIGNUM");
@@ -174,29 +179,42 @@ static int l_bignum_is_bit_set( lua_State *L ) /** bignum_set_bit( BIGNUM bn, nu
   return 1;
 }
 
-static int l_bignum_is_prime( lua_State *L ) /** bignum_is_prime( BIGNUM p, number nchecks ) */
+static int l_bignum_is_prime( lua_State *L ) /** bignum_is_prime( BIGNUM p ) */
 {
   bignum_data_t * p = (bignum_data_t *) luaL_checkudata( L, 1, "BIGNUM" );
-  int nchecks = luaL_optinteger( L, 2, BN_prime_checks );
   BN_CTX * ctx = BN_CTX_new();
-  int is_prime = BN_is_prime_ex( p->bn, nchecks, ctx, NULL );
+  int is_prime =
+#if OPENSSL_API_LEVEL < 30000
+    BN_is_prime_ex( p->bn, BN_prime_checks, ctx, NULL );
+#else
+    BN_check_prime( p->bn, ctx, NULL );
+#endif
   BN_CTX_free( ctx );
   lua_pushboolean( L, is_prime );
   return 1;
 }
 
-static int l_bignum_is_safe_prime( lua_State *L ) /** bignum_is_safe_prime( BIGNUM p, number nchecks ) */
+static int l_bignum_is_safe_prime( lua_State *L ) /** bignum_is_safe_prime( BIGNUM p ) */
 {
   bignum_data_t * p = (bignum_data_t *) luaL_checkudata( L, 1, "BIGNUM" );
-  int nchecks = luaL_optinteger( L, 2, BN_prime_checks );
   BN_CTX * ctx = BN_CTX_new();
-  int is_prime = BN_is_prime_ex( p->bn, nchecks, ctx, NULL );
+  int is_prime =
+#if OPENSSL_API_LEVEL < 30000
+    BN_is_prime_ex( p->bn, BN_prime_checks, ctx, NULL );
+#else
+    BN_check_prime( p->bn, ctx, NULL );
+#endif
   int is_safe = 0;
   if (is_prime) {
     BIGNUM * n = BN_dup( p->bn );
     BN_sub_word( n, (BN_ULONG)1 );
     BN_div_word( n, (BN_ULONG)2 );
-    is_safe = BN_is_prime_ex( n, nchecks, ctx, NULL );
+    is_safe =
+#if OPENSSL_API_LEVEL < 30000
+      BN_is_prime_ex( n, BN_prime_checks, ctx, NULL );
+#else
+      BN_check_prime( n, ctx, NULL );
+#endif
     BN_clear_free( n );
   }
   BN_CTX_free( ctx );
@@ -264,56 +282,12 @@ static int l_rand_pseudo_bytes( lua_State *L ) /** rand_pseudo_bytes( number byt
   unsigned char * result = (unsigned char *) malloc( len );
   if (!result) return luaL_error( L, "Couldn't allocate memory.");
 
-#if OPENSSL_API_COMPAT < 0x10100000L
-  RAND_pseudo_bytes( result, len );
-#else
   get_random_bytes( result, len );
-#endif
   lua_pushlstring( L, (char *) result, len );
   free( result );
   return 1;
 }
 
-static int l_md4(lua_State *L)     /** md4(string s) */
-{
-  size_t len;
-  const unsigned char *s = (unsigned char *) luaL_checklstring( L, 1, &len );
-  unsigned char digest[16];
-
-  lua_pushlstring( L, (char *) MD4( s, len, digest ), 16 );
-  return 1;
-}
-
-static int l_md5(lua_State *L)     /** md5(string s) */
-{
-  size_t len;
-  const unsigned char *s = (unsigned char *) luaL_checklstring( L, 1, &len );
-  unsigned char digest[16];
-
-  lua_pushlstring( L, (char *) MD5( s, len, digest ), 16 );
-  return 1;
-}
-
-static int l_sha1(lua_State *L)     /** sha1(string s) */
-{
-  size_t len;
-  const unsigned char *s = (unsigned char *) luaL_checklstring( L, 1, &len );
-  unsigned char digest[20];
-
-  lua_pushlstring( L, (char *) SHA1( s, len, digest ), 20 );
-  return 1;
-}
-
-static int l_ripemd160(lua_State *L)     /** ripemd160(string s) */
-{
-  size_t len;
-  const unsigned char *s = (unsigned char *) luaL_checklstring( L, 1, &len );
-  unsigned char digest[20];
-
-  lua_pushlstring( L, (char *) RIPEMD160( s, len, digest ), 20 );
-  return 1;
-}
-
 static int l_digest(lua_State *L)     /** digest(string algorithm, string message) */
 {
   size_t msg_len;
@@ -338,9 +312,7 @@ static int l_digest(lua_State *L)     /** digest(string algorithm, string messag
       EVP_DigestUpdate( PASS_EVP_CTX(mdctx), msg, msg_len ) &&
       EVP_DigestFinal_ex( PASS_EVP_CTX(mdctx), digest, &digest_len ))) {
     FUNC_EVP_MD_CTX_cleanup( PASS_EVP_CTX(mdctx) );
-    unsigned long e = ERR_get_error();
+    return NSE_SSL_LUA_ERR(L);
-    return luaL_error( L, "OpenSSL error %d in %s: function %s: %s", e, ERR_lib_error_string(e),
-                       ERR_func_error_string(e), ERR_reason_error_string(e));
   }
   FUNC_EVP_MD_CTX_cleanup( PASS_EVP_CTX(mdctx) );
 
@@ -348,6 +320,20 @@ static int l_digest(lua_State *L)     /** digest(string algorithm, string messag
   return 1;
 }
 
+/** md4(string s) */
+#define NSE_DECLARE_DIGEST(_mdname) \
+static int l_##_mdname(lua_State *L) \
+{ \
+  lua_pushliteral(L, #_mdname); \
+  lua_insert(L, 1); \
+  return l_digest(L); \
+}
+
+NSE_DECLARE_DIGEST(md4)
+NSE_DECLARE_DIGEST(md5)
+NSE_DECLARE_DIGEST(sha1)
+NSE_DECLARE_DIGEST(ripemd160)
+
 static int l_hmac(lua_State *L)     /** hmac(string algorithm, string key, string message) */
 {
   size_t key_len, msg_len;
@@ -432,9 +418,7 @@ static int l_encrypt(lua_State *L) /** encrypt( string algorithm, string key, st
       EVP_EncryptInit_ex( PASS_EVP_CTX(cipher_ctx), evp_cipher, NULL, NULL, NULL ) &&
       EVP_CIPHER_CTX_set_key_length( PASS_EVP_CTX(cipher_ctx), key_len ) &&
       EVP_CIPHER_CTX_set_padding( PASS_EVP_CTX(cipher_ctx), padding ))) {
-    unsigned long e = ERR_get_error();
+    return NSE_SSL_LUA_ERR(L);
-    return luaL_error( L, "OpenSSL error %d in %s: function %s: %s", e, ERR_lib_error_string(e),
-                       ERR_func_error_string(e), ERR_reason_error_string(e));
   }
 
   if (iv != NULL && (int) iv_len != EVP_CIPHER_CTX_iv_length( PASS_EVP_CTX(cipher_ctx) )) {
@@ -452,9 +436,7 @@ static int l_encrypt(lua_State *L) /** encrypt( string algorithm, string key, st
       EVP_EncryptFinal_ex( PASS_EVP_CTX(cipher_ctx), out + out_len, &final_len ) )) {
     FUNC_EVP_CIPHER_CTX_cleanup( PASS_EVP_CTX(cipher_ctx) );
     free( out );
-    unsigned long e = ERR_get_error();
+    return NSE_SSL_LUA_ERR(L);
-    return luaL_error( L, "OpenSSL error %d in %s: function %s: %s", e, ERR_lib_error_string(e),
-                       ERR_func_error_string(e), ERR_reason_error_string(e));
   }
 
   lua_pushlstring( L, (char *) out, out_len + final_len );
@@ -491,9 +473,7 @@ static int l_decrypt(lua_State *L) /** decrypt( string algorithm, string key, st
       EVP_DecryptInit_ex( PASS_EVP_CTX(cipher_ctx), evp_cipher, NULL, NULL, NULL ) &&
       EVP_CIPHER_CTX_set_key_length( PASS_EVP_CTX(cipher_ctx), key_len ) &&
       EVP_CIPHER_CTX_set_padding( PASS_EVP_CTX(cipher_ctx), padding ))) {
-    unsigned long e = ERR_get_error();
+    return NSE_SSL_LUA_ERR(L);
-    return luaL_error( L, "OpenSSL error %d in %s: function %s: %s", e, ERR_lib_error_string(e),
-                       ERR_func_error_string(e), ERR_reason_error_string(e));
   }
 
   if (iv != NULL && (int) iv_len != EVP_CIPHER_CTX_iv_length( PASS_EVP_CTX(cipher_ctx) )) {
@@ -511,9 +491,7 @@ static int l_decrypt(lua_State *L) /** decrypt( string algorithm, string key, st
       EVP_DecryptFinal_ex( PASS_EVP_CTX(cipher_ctx), out + out_len, &final_len ) )) {
     FUNC_EVP_CIPHER_CTX_cleanup( PASS_EVP_CTX(cipher_ctx) );
     free( out );
-    unsigned long e = ERR_get_error();
+    return NSE_SSL_LUA_ERR(L);
-    return luaL_error( L, "OpenSSL error %d in %s: function %s: %s", e, ERR_lib_error_string(e),
-                       ERR_func_error_string(e), ERR_reason_error_string(e));
   }
 
   lua_pushlstring( L, (char *) out, out_len + final_len );
@@ -531,51 +509,21 @@ static int l_DES_string_to_key(lua_State *L) /** DES_string_to_key( string data
   if (len != 7 )
     return luaL_error( L, "String must have length of 7 bytes." );
 
-  DES_cblock key;
+  unsigned char key[8] = {0};
-  key[0] = data[0];
+  // key is each 7 bits of data separated by 0 bit
+  // Clear the lsb of the first byte:
+  key[0] = data[0] & ~1;
+  // Least significant i bits of i-1 byte plus most significant 8-i bits of i-th byte
+  // clearing the lsb of result to keep only the 7-i bits of i-th byte
   for( int i = 1; i < 8; i++ )
-    key[i] = data[i-1] << (8-i) | data[i] >> i;
+    key[i] = (data[i-1] << (8-i) | data[i] >> i) & ~1;
 
-  DES_set_odd_parity( &key ); // lgtm [cpp/weak-cryptographic-algorithm]
+  // DES_set_odd_parity( &key ); // lgtm [cpp/weak-cryptographic-algorithm]
 
   lua_pushlstring( L, (char *) key, 8 );
   return 1;
 }
 
-#ifndef OPENSSL_NO_RC4
-static int l_rc4_options (lua_State *L)
-{
-  lua_pushstring(L, RC4_options());
-  return 1;
-}
-
-static int l_rc4_encrypt (lua_State *L)
-{
-  RC4_KEY *key = (RC4_KEY *) lua_touserdata(L, lua_upvalueindex(1));
-  size_t len;
-  const char *indata = luaL_checklstring(L, 1, &len);
-  unsigned char *outdata = (unsigned char *) lua_newuserdata(L, sizeof(unsigned char)*len);
-
-  RC4(key, len, (const unsigned char *)indata, outdata); // lgtm [cpp/weak-cryptographic-algorithm]
-  lua_pushlstring(L, (const char *)outdata, len);
-
-  return 1;
-}
-
-static int l_rc4 (lua_State *L)
-{
-  size_t len;
-  const char *data = luaL_checklstring(L, 1, &len);
-
-  lua_newuserdata(L, sizeof(RC4_KEY));
-  RC4_set_key((RC4_KEY *)lua_touserdata(L, -1), (int)len, (const unsigned char *)data); // lgtm [cpp/weak-cryptographic-algorithm]
-
-  lua_pushcclosure(L, l_rc4_encrypt, 1);
-
-  return 1;
-}
-#endif
-
 static const struct luaL_Reg bignum_methods[] = {
   { "num_bits", l_bignum_num_bits },
   { "num_bytes", l_bignum_num_bytes },
@@ -604,7 +552,7 @@ static const struct luaL_Reg openssllib[] = {
   { "bignum_dec2bn", l_bignum_dec2bn },
   { "bignum_hex2bn", l_bignum_hex2bn },
   { "bignum_rand", l_bignum_rand },
-  { "bignum_pseudo_rand", l_bignum_pseudo_rand },
+  { "bignum_pseudo_rand", l_bignum_rand },
   { "bignum_bn2bin", l_bignum_bn2bin },
   { "bignum_bn2dec", l_bignum_bn2dec },
   { "bignum_bn2hex", l_bignum_bn2hex },
@@ -613,10 +561,12 @@ static const struct luaL_Reg openssllib[] = {
   { "bignum_div", l_bignum_div },
   { "rand_bytes", l_rand_bytes },
   { "rand_pseudo_bytes", l_rand_pseudo_bytes },
+  // These functions declared above with NSE_DECLARE_DIGEST
   { "md4", l_md4 },
   { "md5", l_md5 },
   { "sha1", l_sha1 },
   { "ripemd160", l_ripemd160 },
+
   { "digest", l_digest },
   { "hmac", l_hmac },
   { "encrypt", l_encrypt },
@@ -624,10 +574,6 @@ static const struct luaL_Reg openssllib[] = {
   { "DES_string_to_key", l_DES_string_to_key },
   { "supported_digests", l_supported_digests },
   { "supported_ciphers", l_supported_ciphers },
-#ifndef OPENSSL_NO_RC4
-  { "rc4_options", l_rc4_options },
-  { "rc4", l_rc4 },
-#endif
   { NULL, NULL }
 };
 
@@ -636,11 +582,15 @@ LUALIB_API int luaopen_openssl(lua_State *L) {
 #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined LIBRESSL_VERSION_NUMBER
   OpenSSL_add_all_algorithms();
   ERR_load_crypto_strings();
-#else
+#elif OPENSSL_API_LEVEL >= 30000
-  /* This is now deprecated in OpenSSL 1.1.0 _ No explicit initialisation
+  if (NULL == OSSL_PROVIDER_load(NULL, "legacy") && o.verbose)
-    or de-initialisation is necessary */
+  {
-  // OpenSSL_add_all_algorithms();
+    log_write(LOG_STDOUT, "%s: OpenSSL legacy provider failed to load.\n", SCRIPT_ENGINE);
-  // ERR_load_crypto_strings();
+  }
+  if (NULL == OSSL_PROVIDER_load(NULL, "default") && o.verbose)
+  {
+    log_write(LOG_STDOUT, "%s: OpenSSL default provider failed to load.\n", SCRIPT_ENGINE);
+  }
 #endif
 
   luaL_newlib(L, openssllib);

nse_ssl_cert.cc

@@ -74,7 +74,6 @@
 #include <openssl/bn.h>
 #include <openssl/bio.h>
 #include <openssl/pem.h>
-#include <openssl/rsa.h>
 #include <openssl/ssl.h>
 #include <openssl/x509.h>
 #include <openssl/x509v3.h>
@@ -90,6 +89,27 @@
 #define X509_get0_notAfter X509_get_notAfter
 #endif
 
+/* OPENSSL_API_LEVEL per OpenSSL 3.0: decimal MMmmpp */
+#ifndef OPENSSL_API_LEVEL
+# if OPENSSL_API_COMPAT < 0x900000L
+#  define OPENSSL_API_LEVEL (OPENSSL_API_COMPAT)
+# else
+#  define OPENSSL_API_LEVEL \
+     (((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000  \
+      + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+      + ((OPENSSL_API_COMPAT >> 12) & 0xFF))
+# endif
+#endif
+
+#if OPENSSL_API_LEVEL >= 30000
+#include <openssl/core_names.h>
+/* Deprecated in OpenSSL 3.0 */
+#define SSL_get_peer_certificate SSL_get1_peer_certificate
+#else
+#include <openssl/rsa.h>
+#endif
+
+
 /* struct tm */
 #include <time.h>
 
@@ -439,7 +459,39 @@ static const char *pkey_type_to_string(int type)
 }
 
 int lua_push_ecdhparams(lua_State *L, EVP_PKEY *pubkey) {
-#ifdef HAVE_OPENSSL_EC
+#if OPENSSL_API_LEVEL >= 30000
+  char tmp[64] = {0};
+  size_t len = 0;
+  /* This structure (ecdhparams.curve_params) comes from tls.lua */
+  lua_createtable(L, 0, 1); /* ecdhparams */
+  lua_createtable(L, 0, 2); /* curve_params */
+  if (EVP_PKEY_get_utf8_string_param(pubkey, OSSL_PKEY_PARAM_GROUP_NAME,
+        tmp, sizeof(tmp), &len)) {
+    lua_pushlstring(L, tmp, len);
+    lua_setfield(L, -2, "curve");
+    lua_pushliteral(L, "namedcurve");
+    lua_setfield(L, -2, "ec_curve_type");
+  }
+  else if (EVP_PKEY_get_utf8_string_param(pubkey, OSSL_PKEY_PARAM_EC_FIELD_TYPE,
+        tmp, sizeof(tmp), &len)) {
+    /* According to RFC 5480 section 2.1.1, explicit curves must not be used with
+       X.509. This may change in the future, but for now it doesn't seem worth it
+       to add in code to extract the extra parameters. */
+    if (0 == strncmp(tmp, "prime-field", len)) {
+      lua_pushliteral(L, "explicit_prime");
+    }
+    else if (0 == strncmp(tmp, "characteristic-two-field", len)) {
+      lua_pushliteral(L, "explicit_char2");
+    }
+    else {
+      /* Something weird happened. */
+      lua_pushlstring(L, tmp, len);
+    }
+    lua_setfield(L, -2, "ec_curve_type");
+  }
+  lua_setfield(L, -2, "curve_params");
+  return 1;
+#elif defined(HAVE_OPENSSL_EC)
   EC_KEY *ec_key = EVP_PKEY_get1_EC_KEY(pubkey);
   const EC_GROUP *group = EC_KEY_get0_group(ec_key);
   int nid;
@@ -582,16 +634,24 @@ static int parse_ssl_cert(lua_State *L, X509 *cert)
   else
 #endif
   if (pkey_type == EVP_PKEY_RSA) {
+#if OPENSSL_API_LEVEL < 30000
     RSA *rsa = EVP_PKEY_get1_RSA(pubkey);
     if (rsa) {
+#endif
       /* exponent */
       bignum_data_t * data = (bignum_data_t *) lua_newuserdata( L, sizeof(bignum_data_t));
       luaL_getmetatable( L, "BIGNUM" );
       lua_setmetatable( L, -2 );
-      data->should_free = false;
 #if HAVE_OPAQUE_STRUCTS
-      const BIGNUM *n, *e;
+      BIGNUM *n = NULL, *e = NULL;
+#if OPENSSL_API_LEVEL < 30000
+      data->should_free = false;
       RSA_get0_key(rsa, &n, &e, NULL);
+#else
+      data->should_free = true;
+      EVP_PKEY_get_bn_param(pubkey, OSSL_PKEY_PARAM_RSA_E, &e);
+      EVP_PKEY_get_bn_param(pubkey, OSSL_PKEY_PARAM_RSA_N, &n);
+#endif
       data->bn = (BIGNUM*) e;
 #else
       data->bn = rsa->e;
@@ -601,15 +661,21 @@ static int parse_ssl_cert(lua_State *L, X509 *cert)
       data = (bignum_data_t *) lua_newuserdata( L, sizeof(bignum_data_t));
       luaL_getmetatable( L, "BIGNUM" );
       lua_setmetatable( L, -2 );
-      data->should_free = false;
 #if HAVE_OPAQUE_STRUCTS
+#if OPENSSL_API_LEVEL < 30000
+      data->should_free = false;
+#else
+      data->should_free = true;
+#endif
       data->bn = (BIGNUM*) n;
 #else
       data->bn = rsa->n;
 #endif
       lua_setfield(L, -2, "modulus");
+#if OPENSSL_API_LEVEL < 30000
       RSA_free(rsa);
     }
+#endif
   }
   lua_pushstring(L, pkey_type_to_string(pkey_type));
   lua_setfield(L, -2, "type");

nselib/openssl.luadoc

@@ -52,20 +52,20 @@ function bignum_is_bit_set(bignum, position)
 --
 -- Performs Miller-Rabin probabilistic primality tests.
 -- @param bignum bignum to check for primality
--- @param nchecks Number of checks to perform. Default: number of checks dependent on bitsize of bignum, with a false positive rate of at most 2^-80
+-- @return True if the number is probably prime with a false positive rate of at most 2^-80, false if it is composite.
--- @return True if the number is probably prime, false if it is composite.
+
-function bignum_is_prime(bignum, nchecks)
+function bignum_is_prime(bignum)
 
 --- Checks whether <code>bignum</code> is a safe prime.
 --
 -- A safe prime is defined as a prime p so that (p-1)/2 is also prime. Using
 -- non-safe primes in discrete logarithm cryptography like Diffie-Hellman can
--- result in weak, easily broken key exchanges.
+-- result in weak, easily broken key exchanges. The number of checks is
+-- dependent on bitsize of bignum, with a false positive rate of at most 2^-80
 -- @param bignum bignum to check for primality
--- @param nchecks Number of checks to perform. Default: number of checks dependent on bitsize of bignum, with a false positive rate of at most 2^-80
 -- @return True if the number is a safe prime, false if it is not.
 -- @return True if the number is probably prime, false if it is composite.
-function bignum_is_safe_prime(bignum, nchecks)
+function bignum_is_safe_prime(bignum)
 
 --- Converts a binary-encoded string into a bignum.
 -- @param string Binary string.
@@ -103,8 +103,10 @@ function bignum_bn2hex(bignum)
 function bignum_rand(bits)
 
 --- Returns a pseudorandom bignum.
+--
+-- Alias for <code>bignum_rand()</code>.
 -- @param bits Size of the returned bignum in bits.
--- @return Pseudorandom bignum.
+-- @return Random bignum.
 function bignum_pseudo_rand(bits)
 
 --- Returns the bignum which is the result of <code>a</code>^<code>p</code> mod
@@ -208,12 +210,3 @@ function supported_digests()
 -- @param data A 7-byte string.
 -- @return An 8-byte string.
 function DES_string_to_key(data)
-
---- Returns options for RC4.
--- @return Option string.
-function rc4_options()
-
---- Function which generates an RC4 cipher function with the given key.
--- @param key_data The key for the cipher.
--- @return A function which performs the RC4 stream cipher on an input string and returns the result.
-function rc4(key_data)

nsock/src/nsock_ssl.c

@@ -59,10 +59,14 @@
 
 #include "nsock.h"
 #include "nsock_internal.h"
+#include "nsock_log.h"
 #include "nsock_ssl.h"
 #include "netutils.h"
 
 #if HAVE_OPENSSL
+#if OPENSSL_API_LEVEL >= 30000
+#include <openssl/provider.h>
+#endif
 
 /* Disallow anonymous ciphers (Diffie-Hellman key agreement), low bit-strength
  * ciphers, export-crippled ciphers, and MD5. Prefer ciphers in decreasing order
@@ -85,6 +89,15 @@ static SSL_CTX *ssl_init_helper(const SSL_METHOD *method) {
 #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined LIBRESSL_VERSION_NUMBER
   SSL_load_error_strings();
   SSL_library_init();
+#elif OPENSSL_API_LEVEL >= 30000
+  if (NULL == OSSL_PROVIDER_load(NULL, "legacy"))
+  {
+    nsock_log_error("OpenSSL legacy provider failed to load.\n");
+  }
+  if (NULL == OSSL_PROVIDER_load(NULL, "default"))
+  {
+    nsock_log_error("OpenSSL default provider failed to load.\n");
+  }
 #endif
 
   ctx = SSL_CTX_new(method);

nsock/src/nsock_ssl.h

@@ -69,6 +69,24 @@
 #include <openssl/err.h>
 #include <openssl/rand.h>
 
+/* OPENSSL_API_LEVEL per OpenSSL 3.0: decimal MMmmpp */
+#ifndef OPENSSL_API_LEVEL
+# if OPENSSL_API_COMPAT < 0x900000L
+#  define OPENSSL_API_LEVEL (OPENSSL_API_COMPAT)
+# else
+#  define OPENSSL_API_LEVEL \
+     (((OPENSSL_API_COMPAT >> 28) & 0xF) * 10000  \
+      + ((OPENSSL_API_COMPAT >> 20) & 0xFF) * 100 \
+      + ((OPENSSL_API_COMPAT >> 12) & 0xFF))
+# endif
+#endif
+
+
+#if OPENSSL_API_LEVEL >= 30000
+/* Deprecated in OpenSSL 3.0 */
+#define SSL_get_peer_certificate SSL_get1_peer_certificate
+#endif
+
 struct sslinfo {
   /* SSL_ERROR_NONE, SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_READ, or
    * SSL_ERROR_WANT_WRITE */