Validate OpenSSL libraries even in custom locations

Fixes #2420, closes #2436

CHANGELOG

@@ -1,5 +1,8 @@
 #Nmap Changelog ($Id$); -*-text-*-
 
+o [GH#2420][GH#2436] Support for EC crypto was not properly enabled if Nmap
+  was compiled with OpenSSL in a custom location. [nnposter]
+
 o [GH#2388] Fix TDS7 password encoding for mssql.lua, which had been assuming
   ASCII input even though other parts of the library had been passing it Unicode.
 

configure

@@ -5893,6 +5893,64 @@ fi
 
 
 
+# OpenSSL and NSE C modules can require dlopen
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing dlopen" >&5
+$as_echo_n "checking for library containing dlopen... " >&6; }
+if ${ac_cv_search_dlopen+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char dlopen ();
+int
+main ()
+{
+return dlopen ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' dl; do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_search_dlopen=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext
+  if ${ac_cv_search_dlopen+:} false; then :
+  break
+fi
+done
+if ${ac_cv_search_dlopen+:} false; then :
+
+else
+  ac_cv_search_dlopen=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_dlopen" >&5
+$as_echo "$ac_cv_search_dlopen" >&6; }
+ac_res=$ac_cv_search_dlopen
+if test "$ac_res" != no; then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
+fi
+
+
 # We test whether they specified openssl desires explicitly
 use_openssl="yes"
 specialssldir=""
@@ -5917,12 +5975,15 @@ fi
 
 
 # If they didn't specify it, we try to find it
-if test "$use_openssl" = "yes" -a -z "$specialssldir"; then
+if test "$use_openssl" = "yes"; then
   ac_fn_c_check_header_mongrel "$LINENO" "openssl/ssl.h" "ac_cv_header_openssl_ssl_h" "$ac_includes_default"
 if test "x$ac_cv_header_openssl_ssl_h" = xyes; then :
 
 else
    use_openssl="no"
+    if test -n "$specialssldir"; then
+      as_fn_error $? "Specific OpenSSL location was requested but openssl/ssl.h was not found. Try correcting the --with-openssl=DIR argument." "$LINENO" 5
+    fi
     if test "$with_openssl" = "yes"; then
       as_fn_error $? "OpenSSL was explicitly requested but openssl/ssl.h was not found. Try the --with-openssl=DIR argument to give the location of OpenSSL or run configure with --without-openssl." "$LINENO" 5
     fi
@@ -5932,6 +5993,7 @@ $as_echo "$as_me: WARNING: Failed to find openssl/ssl.h so OpenSSL will not be u
 fi
 
 
+fi
 
 # use_openssl="yes" given explicitly in next 2 rules to avoid adding lib to $LIBS
 if test "$use_openssl" = "yes"; then
@@ -5975,6 +6037,9 @@ if test "x$ac_cv_lib_crypto_BIO_int_ctrl" = xyes; then :
    use_openssl="yes"
 else
    use_openssl="no"
+      if test -n "$specialssldir"; then
+        as_fn_error $? "Specific OpenSSL location was requested but libcrypto was not found. Try correcting the --with-openssl=DIR argument." "$LINENO" 5
+      fi
       if test "$with_openssl" = "yes"; then
         as_fn_error $? "OpenSSL was explicitly requested but libcrypto was not found. Try the --with-openssl=DIR argument to give the location of OpenSSL or run configure with --without-openssl." "$LINENO" 5
       fi
@@ -6026,11 +6091,15 @@ if test "x$ac_cv_lib_ssl_SSL_new" = xyes; then :
    use_openssl="yes"
 else
    use_openssl="no"
+      if test -n "$specialssldir"; then
+        as_fn_error $? "Specific OpenSSL location was requested but libssl was not found. Try correcting the --with-openssl=DIR argument." "$LINENO" 5
+      fi
       if test "$with_openssl" = "yes"; then
         as_fn_error $? "OpenSSL was explicitly requested but libssl was not found. Try the --with-openssl=DIR argument to give the location of OpenSSL or run configure with --without-openssl." "$LINENO" 5
       fi
       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Failed to find libssl so OpenSSL will not be used. If it is installed you can try the --with-openssl=DIR argument" >&5
 $as_echo "$as_me: WARNING: Failed to find libssl so OpenSSL will not be used. If it is installed you can try the --with-openssl=DIR argument" >&2;}
+
 fi
 
 fi
@@ -6082,7 +6151,6 @@ $as_echo "$as_me: WARNING: Disabling support for EC crypto" >&2;}
 fi
 
 fi
-fi
 
 OPENSSL_LIBS=
 if test "$use_openssl" = "yes"; then
@@ -6999,64 +7067,6 @@ $as_echo "#define LUA_INCLUDED 1" >>confdefs.h
 fi
 
 
-# OpenSSL and NSE C modules can require dlopen
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing dlopen" >&5
-$as_echo_n "checking for library containing dlopen... " >&6; }
-if ${ac_cv_search_dlopen+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-  ac_func_search_save_LIBS=$LIBS
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-
-/* Override any GCC internal prototype to avoid an error.
-   Use char because int might match the return type of a GCC
-   builtin and then its argument prototype would still apply.  */
-#ifdef __cplusplus
-extern "C"
-#endif
-char dlopen ();
-int
-main ()
-{
-return dlopen ();
-  ;
-  return 0;
-}
-_ACEOF
-for ac_lib in '' dl; do
-  if test -z "$ac_lib"; then
-    ac_res="none required"
-  else
-    ac_res=-l$ac_lib
-    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
-  fi
-  if ac_fn_c_try_link "$LINENO"; then :
-  ac_cv_search_dlopen=$ac_res
-fi
-rm -f core conftest.err conftest.$ac_objext \
-    conftest$ac_exeext
-  if ${ac_cv_search_dlopen+:} false; then :
-  break
-fi
-done
-if ${ac_cv_search_dlopen+:} false; then :
-
-else
-  ac_cv_search_dlopen=no
-fi
-rm conftest.$ac_ext
-LIBS=$ac_func_search_save_LIBS
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_dlopen" >&5
-$as_echo "$ac_cv_search_dlopen" >&6; }
-ac_res=$ac_cv_search_dlopen
-if test "$ac_res" != no; then :
-  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
-
-fi
-
-
 # They don't want lua
 if test "$no_lua" = "yes"; then
   trace_no_use="$trace_no_use lua"

configure.ac

@@ -342,6 +342,9 @@ AC_SUBST(UNINSTALLNPING)
 AC_SUBST(NPING_CLEAN)
 AC_SUBST(NPING_DIST_CLEAN)
 
+# OpenSSL and NSE C modules can require dlopen
+AC_SEARCH_LIBS(dlopen, dl)
+
 # We test whether they specified openssl desires explicitly
 use_openssl="yes"
 specialssldir=""
@@ -363,20 +366,27 @@ AC_HELP_STRING([--with-openssl=DIR],[Use optional openssl libs and includes from
 )
 
 # If they didn't specify it, we try to find it
-if test "$use_openssl" = "yes" -a -z "$specialssldir"; then
+if test "$use_openssl" = "yes"; then
   AC_CHECK_HEADER(openssl/ssl.h,,
   [ use_openssl="no"
+    if test -n "$specialssldir"; then
+      AC_MSG_ERROR([Specific OpenSSL location was requested but openssl/ssl.h was not found. Try correcting the --with-openssl=DIR argument.])
+    fi
     if test "$with_openssl" = "yes"; then
       AC_MSG_ERROR([OpenSSL was explicitly requested but openssl/ssl.h was not found. Try the --with-openssl=DIR argument to give the location of OpenSSL or run configure with --without-openssl.])
     fi
     AC_MSG_WARN([Failed to find openssl/ssl.h so OpenSSL will not be used. If it is installed you can try the --with-openssl=DIR argument])
   ])
+fi
 
 # use_openssl="yes" given explicitly in next 2 rules to avoid adding lib to $LIBS
 if test "$use_openssl" = "yes"; then
   AC_CHECK_LIB(crypto, BIO_int_ctrl,
     [ use_openssl="yes"],
     [ use_openssl="no"
+      if test -n "$specialssldir"; then
+        AC_MSG_ERROR([Specific OpenSSL location was requested but libcrypto was not found. Try correcting the --with-openssl=DIR argument.])
+      fi
       if test "$with_openssl" = "yes"; then
         AC_MSG_ERROR([OpenSSL was explicitly requested but libcrypto was not found. Try the --with-openssl=DIR argument to give the location of OpenSSL or run configure with --without-openssl.])
       fi
@@ -388,10 +398,14 @@ if test "$use_openssl" = "yes" -a -z "$specialssldir"; then
   AC_CHECK_LIB(ssl, SSL_new,
     [ use_openssl="yes" ],
     [ use_openssl="no"
+      if test -n "$specialssldir"; then
+        AC_MSG_ERROR([Specific OpenSSL location was requested but libssl was not found. Try correcting the --with-openssl=DIR argument.])
+      fi
       if test "$with_openssl" = "yes"; then
         AC_MSG_ERROR([OpenSSL was explicitly requested but libssl was not found. Try the --with-openssl=DIR argument to give the location of OpenSSL or run configure with --without-openssl.])
       fi
-    AC_MSG_WARN([Failed to find libssl so OpenSSL will not be used. If it is installed you can try the --with-openssl=DIR argument]) ],
+      AC_MSG_WARN([Failed to find libssl so OpenSSL will not be used. If it is installed you can try the --with-openssl=DIR argument])
+    ],
     [ -lcrypto ])
 fi
 
@@ -400,7 +414,6 @@ if test "$use_openssl" = "yes" -a -z "$specialssldir"; then
    [AC_DEFINE(HAVE_OPENSSL_EC, 1, [Have EVP_PKEY_get1_EC_KEY])],
    [AC_MSG_WARN([Disabling support for EC crypto])])
 fi
-fi
 
 OPENSSL_LIBS=
 if test "$use_openssl" = "yes"; then
@@ -808,9 +821,6 @@ AC_HELP_STRING([--without-liblua], [Compile without lua (this will exclude all o
   esac]
 )
 
-# OpenSSL and NSE C modules can require dlopen
-AC_SEARCH_LIBS(dlopen, dl)
-
 # They don't want lua
 if test "$no_lua" = "yes"; then
   trace_no_use="$trace_no_use lua"