Support OpenSSL 1.1.0.

Changes needed to build at all:

- Check for SSL_new in configure, now that SSL_library_init is a macro.
- Do not access struct members directly. This includes some new code in
  pgcrypto, to use the resource owner mechanism to ensure that we don't
  leak OpenSSL handles, now that we can't embed them in other structs
  anymore.
- RAND_SSLeay() -> RAND_OpenSSL()

Changes that were needed to silence deprecation warnings, but were not
strictly necessary:

- RAND_pseudo_bytes() -> RAND_bytes().
- SSL_library_init() and OpenSSL_config() -> OPENSSL_init_ssl()
- ASN1_STRING_data() -> ASN1_STRING_get0_data()
- DH_generate_parameters() -> DH_generate_parameters()
- Locking callbacks are not needed with OpenSSL 1.1.0 anymore. (Good
  riddance!)

Also change references to SSLEAY_VERSION_NUMBER with OPENSSL_VERSION_NUMBER,
for the sake of consistency. OPENSSL_VERSION_NUMBER has existed since time
immemorial.

Fix SSL test suite to work with OpenSSL 1.1.0. CA certificates must have
the "CA:true" basic constraint extension now, or OpenSSL will refuse them.
Regenerate the test certificates with that. The "openssl" binary, used to
generate the certificates, is also now more picky, and throws an error
if an X509 extension is specified in "req_extensions", but that section
is empty.

Backpatch to 9.5 and 9.6, per popular demand. The file structure was
somewhat different in earlier branches, so I didn't bother to go further
than that. In back-branches, we still support OpenSSL 0.9.7 and above.
OpenSSL 0.9.6 should still work too, but I didn't test it. In master, we
only support 0.9.8 and above.

Patch by Andreas Karlsson, with additional changes by me.

Discussion: <20160627151604.GD1051@msg.df7cb.de>

6 files changed, 106 insertions, 44 deletions

diff --git a/contrib/pgcrypto/internal.c b/contrib/pgcrypto/internal.c
index cb8ba2633d5..02ff976c25a 100644
--- a/contrib/pgcrypto/internal.c
+++ b/contrib/pgcrypto/internal.c
@@ -620,15 +620,6 @@ px_find_cipher(const char *name, PX_Cipher **res)
  * Randomness provider
  */
 
-/*
- * Use always strong randomness.
- */
-int
-px_get_pseudo_random_bytes(uint8 *dst, unsigned count)
-{
-	return px_get_random_bytes(dst, count);
-}
-
 static time_t seed_time = 0;
 static time_t check_time = 0;
 
diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c
index 976af705915..cee2afe3e55 100644
--- a/contrib/pgcrypto/openssl.c
+++ b/contrib/pgcrypto/openssl.c
@@ -40,6 +40,9 @@
 #include <openssl/rand.h>
 #include <openssl/err.h>
 
+#include "utils/memutils.h"
+#include "utils/resowner.h"
+
 /*
  * Max lengths we might want to handle.
  */
@@ -199,18 +202,73 @@ compat_find_digest(const char *name, PX_MD **res)
  * Hashes
  */
 
+/*
+ * To make sure we don't leak OpenSSL handles on abort, we keep OSSLDigest
+ * objects in a linked list, allocated in TopMemoryContext. We use the
+ * ResourceOwner mechanism to free them on abort.
+ */
 typedef struct OSSLDigest
 {
 	const EVP_MD *algo;
-	EVP_MD_CTX	ctx;
+	EVP_MD_CTX *ctx;
+
+	ResourceOwner owner;
+	struct OSSLDigest *next;
+	struct OSSLDigest *prev;
 } OSSLDigest;
 
+static OSSLDigest *open_digests = NULL;
+static bool resowner_callback_registered = false;
+
+static void
+free_openssldigest(OSSLDigest *digest)
+{
+	EVP_MD_CTX_destroy(digest->ctx);
+	if (digest->prev)
+		digest->prev->next = digest->next;
+	else
+		open_digests = digest->next;
+	if (digest->next)
+		digest->next->prev = digest->prev;
+	pfree(digest);
+}
+
+/*
+ * Close any open OpenSSL handles on abort.
+ */
+static void
+digest_free_callback(ResourceReleasePhase phase,
+					 bool isCommit,
+					 bool isTopLevel,
+					 void *arg)
+{
+	OSSLDigest *curr;
+	OSSLDigest *next;
+
+	if (phase != RESOURCE_RELEASE_AFTER_LOCKS)
+		return;
+
+	next = open_digests;
+	while (next)
+	{
+		curr = next;
+		next = curr->next;
+
+		if (curr->owner == CurrentResourceOwner)
+		{
+			if (isCommit)
+				elog(WARNING, "pgcrypto digest reference leak: digest %p still referenced", curr);
+			free_openssldigest(curr);
+		}
+	}
+}
+
 static unsigned
 digest_result_size(PX_MD *h)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	return EVP_MD_CTX_size(&digest->ctx);
+	return EVP_MD_CTX_size(digest->ctx);
 }
 
 static unsigned
@@ -218,7 +276,7 @@ digest_block_size(PX_MD *h)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	return EVP_MD_CTX_block_size(&digest->ctx);
+	return EVP_MD_CTX_block_size(digest->ctx);
 }
 
 static void
@@ -226,7 +284,7 @@ digest_reset(PX_MD *h)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	EVP_DigestInit_ex(&digest->ctx, digest->algo, NULL);
+	EVP_DigestInit_ex(digest->ctx, digest->algo, NULL);
 }
 
 static void
@@ -234,7 +292,7 @@ digest_update(PX_MD *h, const uint8 *data, unsigned dlen)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	EVP_DigestUpdate(&digest->ctx, data, dlen);
+	EVP_DigestUpdate(digest->ctx, data, dlen);
 }
 
 static void
@@ -242,7 +300,7 @@ digest_finish(PX_MD *h, uint8 *dst)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	EVP_DigestFinal_ex(&digest->ctx, dst, NULL);
+	EVP_DigestFinal_ex(digest->ctx, dst, NULL);
 }
 
 static void
@@ -250,9 +308,7 @@ digest_free(PX_MD *h)
 {
 	OSSLDigest *digest = (OSSLDigest *) h->p.ptr;
 
-	EVP_MD_CTX_cleanup(&digest->ctx);
-
-	px_free(digest);
+	free_openssldigest(digest);
 	px_free(h);
 }
 
@@ -264,6 +320,7 @@ int
 px_find_digest(const char *name, PX_MD **res)
 {
 	const EVP_MD *md;
+	EVP_MD_CTX *ctx;
 	PX_MD	   *h;
 	OSSLDigest *digest;
 
@@ -273,17 +330,43 @@ px_find_digest(const char *name, PX_MD **res)
 		OpenSSL_add_all_algorithms();
 	}
 
+	if (!resowner_callback_registered)
+	{
+		RegisterResourceReleaseCallback(digest_free_callback, NULL);
+		resowner_callback_registered = true;
+	}
+
 	md = EVP_get_digestbyname(name);
 	if (md == NULL)
 		return compat_find_digest(name, res);
 
-	digest = px_alloc(sizeof(*digest));
-	digest->algo = md;
+	/*
+	 * Create an OSSLDigest object, an OpenSSL MD object, and a PX_MD object.
+	 * The order is crucial, to make sure we don't leak anything on
+	 * out-of-memory or other error.
+	 */
+	digest = MemoryContextAlloc(TopMemoryContext, sizeof(*digest));
 
-	EVP_MD_CTX_init(&digest->ctx);
-	if (EVP_DigestInit_ex(&digest->ctx, digest->algo, NULL) == 0)
+	ctx = EVP_MD_CTX_create();
+	if (!ctx)
+	{
+		pfree(digest);
 		return -1;
+	}
+	if (EVP_DigestInit_ex(ctx, md, NULL) == 0)
+	{
+		pfree(digest);
+		return -1;
+	}
+
+	digest->algo = md;
+	digest->ctx = ctx;
+	digest->owner = CurrentResourceOwner;
+	digest->next = open_digests;
+	digest->prev = NULL;
+	open_digests = digest;
 
+	/* The PX_MD object is allocated in the current memory context. */
 	h = px_alloc(sizeof(*h));
 	h->result_size = digest_result_size;
 	h->block_size = digest_block_size;
@@ -979,6 +1062,10 @@ px_find_cipher(const char *name, PX_Cipher **res)
 
 static int	openssl_random_init = 0;
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+#define RAND_OpenSSL RAND_SSLeay
+#endif
+
 /*
  * OpenSSL random should re-feeded occasionally. From /dev/urandom
  * preferably.
@@ -987,7 +1074,7 @@ static void
 init_openssl_rand(void)
 {
 	if (RAND_get_rand_method() == NULL)
-		RAND_set_rand_method(RAND_SSLeay());
+		RAND_set_rand_method(RAND_OpenSSL());
 	openssl_random_init = 1;
 }
 
@@ -1007,21 +1094,6 @@ px_get_random_bytes(uint8 *dst, unsigned count)
 }
 
 int
-px_get_pseudo_random_bytes(uint8 *dst, unsigned count)
-{
-	int			res;
-
-	if (!openssl_random_init)
-		init_openssl_rand();
-
-	res = RAND_pseudo_bytes(dst, count);
-	if (res == 0 || res == 1)
-		return count;
-
-	return PXE_OSSL_RAND_ERROR;
-}
-
-int
 px_add_entropy(const uint8 *data, unsigned count)
 {
 	/*
diff --git a/contrib/pgcrypto/pgcrypto.c b/contrib/pgcrypto/pgcrypto.c
index 2d446d8cc95..27b96c7cc4a 100644
--- a/contrib/pgcrypto/pgcrypto.c
+++ b/contrib/pgcrypto/pgcrypto.c
@@ -454,7 +454,7 @@ pg_random_uuid(PG_FUNCTION_ARGS)
 	int			err;
 
 	/* generate random bits */
-	err = px_get_pseudo_random_bytes(buf, UUID_LEN);
+	err = px_get_random_bytes(buf, UUID_LEN);
 	if (err < 0)
 		ereport(ERROR,
 				(errcode(ERRCODE_EXTERNAL_ROUTINE_INVOCATION_EXCEPTION),
diff --git a/contrib/pgcrypto/pgp-s2k.c b/contrib/pgcrypto/pgp-s2k.c
index 193dd95173f..0bcb95ae8c4 100644
--- a/contrib/pgcrypto/pgp-s2k.c
+++ b/contrib/pgcrypto/pgp-s2k.c
@@ -222,13 +222,13 @@ pgp_s2k_fill(PGP_S2K *s2k, int mode, int digest_algo)
 		case 0:
 			break;
 		case 1:
-			res = px_get_pseudo_random_bytes(s2k->salt, PGP_S2K_SALT);
+			res = px_get_random_bytes(s2k->salt, PGP_S2K_SALT);
 			break;
 		case 3:
-			res = px_get_pseudo_random_bytes(s2k->salt, PGP_S2K_SALT);
+			res = px_get_random_bytes(s2k->salt, PGP_S2K_SALT);
 			if (res < 0)
 				break;
-			res = px_get_pseudo_random_bytes(&tmp, 1);
+			res = px_get_random_bytes(&tmp, 1);
 			if (res < 0)
 				break;
 			s2k->iter = decide_count(tmp);
diff --git a/contrib/pgcrypto/px-crypt.c b/contrib/pgcrypto/px-crypt.c
index e3246fc5b9d..3d423938502 100644
--- a/contrib/pgcrypto/px-crypt.c
+++ b/contrib/pgcrypto/px-crypt.c
@@ -153,7 +153,7 @@ px_gen_salt(const char *salt_type, char *buf, int rounds)
 			return PXE_BAD_SALT_ROUNDS;
 	}
 
-	res = px_get_pseudo_random_bytes((uint8 *) rbuf, g->input_len);
+	res = px_get_random_bytes((uint8 *) rbuf, g->input_len);
 	if (res < 0)
 		return res;
 
diff --git a/contrib/pgcrypto/px.h b/contrib/pgcrypto/px.h
index 0f6bbd7a8d5..9174e137dba 100644
--- a/contrib/pgcrypto/px.h
+++ b/contrib/pgcrypto/px.h
@@ -190,7 +190,6 @@ int			px_find_cipher(const char *name, PX_Cipher **res);
 int			px_find_combo(const char *name, PX_Combo **res);
 
 int			px_get_random_bytes(uint8 *dst, unsigned count);
-int			px_get_pseudo_random_bytes(uint8 *dst, unsigned count);
 int			px_add_entropy(const uint8 *data, unsigned count);
 
 unsigned	px_acquire_system_randomness(uint8 *dst);