Another attempt at fixing build with various OpenSSL versions

It seems we can't easily work around the lack of
X509_get_signature_nid(), so revert the previous attempts and just
disable the tls-server-end-point feature if we don't have it.

5 files changed, 24 insertions, 9 deletions

diff --git a/configure b/configure
index d88863e50cf..45221e1ea3b 100755
--- a/configure
+++ b/configure
@@ -10125,12 +10125,13 @@ else
 fi
 
   fi
-  for ac_func in SSL_get_current_compression
+  for ac_func in SSL_get_current_compression X509_get_signature_nid
 do :
-  ac_fn_c_check_func "$LINENO" "SSL_get_current_compression" "ac_cv_func_SSL_get_current_compression"
-if test "x$ac_cv_func_SSL_get_current_compression" = xyes; then :
+  as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
+ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
+if eval test \"x\$"$as_ac_var"\" = x"yes"; then :
   cat >>confdefs.h <<_ACEOF
-#define HAVE_SSL_GET_CURRENT_COMPRESSION 1
+#define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1
 _ACEOF
 
 fi
diff --git a/configure.in b/configure.in
index 4968b67bf92..4d260345792 100644
--- a/configure.in
+++ b/configure.in
@@ -1064,7 +1064,7 @@ if test "$with_openssl" = yes ; then
      AC_SEARCH_LIBS(CRYPTO_new_ex_data, [eay32 crypto], [], [AC_MSG_ERROR([library 'eay32' or 'crypto' is required for OpenSSL])])
      AC_SEARCH_LIBS(SSL_new, [ssleay32 ssl], [], [AC_MSG_ERROR([library 'ssleay32' or 'ssl' is required for OpenSSL])])
   fi
-  AC_CHECK_FUNCS([SSL_get_current_compression])
+  AC_CHECK_FUNCS([SSL_get_current_compression X509_get_signature_nid])
   # Functions introduced in OpenSSL 1.1.0. We used to check for
   # OPENSSL_VERSION_NUMBER, but that didn't work with 1.1.0, because LibreSSL
   # defines OPENSSL_VERSION_NUMBER to claim version 2.0.0, even though it
diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c
index dff61776bd8..c2032c2f30e 100644
--- a/src/backend/libpq/be-secure-openssl.c
+++ b/src/backend/libpq/be-secure-openssl.c
@@ -57,7 +57,6 @@
 #ifndef OPENSSL_NO_ECDH
 #include <openssl/ec.h>
 #endif
-#include <openssl/x509.h>
 
 #include "libpq/libpq.h"
 #include "miscadmin.h"
@@ -1250,6 +1249,7 @@ be_tls_get_peer_finished(Port *port, size_t *len)
 char *
 be_tls_get_certificate_hash(Port *port, size_t *len)
 {
+#ifdef HAVE_X509_GET_SIGNATURE_NID
 	X509	   *server_cert;
 	char	   *cert_hash;
 	const EVP_MD *algo_type = NULL;
@@ -1266,7 +1266,7 @@ be_tls_get_certificate_hash(Port *port, size_t *len)
 	 * Get the signature algorithm of the certificate to determine the
 	 * hash algorithm to use for the result.
 	 */
-	if (!OBJ_find_sigid_algs(OBJ_obj2nid(server_cert->sig_alg->algorithm),
+	if (!OBJ_find_sigid_algs(X509_get_signature_nid(server_cert),
 							 &algo_nid, NULL))
 		elog(ERROR, "could not determine server certificate signature algorithm");
 
@@ -1299,6 +1299,12 @@ be_tls_get_certificate_hash(Port *port, size_t *len)
 	*len = hash_size;
 
 	return cert_hash;
+#else
+	ereport(ERROR,
+			(errcode(ERRCODE_PROTOCOL_VIOLATION),
+			 errmsg("channel binding type \"tls-server-end-point\" is not supported by this build")));
+	return NULL;
+#endif
 }
 
 /*
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 27b13687211..f98f773ff02 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -681,6 +681,9 @@
 /* Define to 1 if you have the <winldap.h> header file. */
 #undef HAVE_WINLDAP_H
 
+/* Define to 1 if you have the `X509_get_signature_nid' function. */
+#undef HAVE_X509_GET_SIGNATURE_NID
+
 /* Define to 1 if your compiler understands __builtin_bswap16. */
 #undef HAVE__BUILTIN_BSWAP16
 
diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c
index ecd68061a2e..b50bfd144a1 100644
--- a/src/interfaces/libpq/fe-secure-openssl.c
+++ b/src/interfaces/libpq/fe-secure-openssl.c
@@ -58,7 +58,6 @@
 #ifdef USE_SSL_ENGINE
 #include <openssl/engine.h>
 #endif
-#include <openssl/x509.h>
 #include <openssl/x509v3.h>
 
 static bool verify_peer_name_matches_certificate(PGconn *);
@@ -430,6 +429,7 @@ pgtls_get_finished(PGconn *conn, size_t *len)
 char *
 pgtls_get_peer_certificate_hash(PGconn *conn, size_t *len)
 {
+#ifdef HAVE_X509_GET_SIGNATURE_NID
 	X509	   *peer_cert;
 	const EVP_MD *algo_type;
 	unsigned char hash[EVP_MAX_MD_SIZE];	/* size for SHA-512 */
@@ -448,7 +448,7 @@ pgtls_get_peer_certificate_hash(PGconn *conn, size_t *len)
 	 * Get the signature algorithm of the certificate to determine the hash
 	 * algorithm to use for the result.
 	 */
-	if (!OBJ_find_sigid_algs(OBJ_obj2nid(peer_cert->sig_alg->algorithm),
+	if (!OBJ_find_sigid_algs(X509_get_signature_nid(peer_cert),
 							 &algo_nid, NULL))
 	{
 		printfPQExpBuffer(&conn->errorMessage,
@@ -499,6 +499,11 @@ pgtls_get_peer_certificate_hash(PGconn *conn, size_t *len)
 	*len = hash_size;
 
 	return cert_hash;
+#else
+	printfPQExpBuffer(&conn->errorMessage,
+					  libpq_gettext("channel binding type \"tls-server-end-point\" is not supported by this build\n"));
+	return NULL;
+#endif
 }
 
 /* ------------------------------------------------------------ */