Improve error handling of cryptohash computations

The existing cryptohash facility was causing problems in some code paths
related to MD5 (frontend and backend) that relied on the fact that the
only type of error that could happen would be an OOM, as the MD5
implementation used in PostgreSQL ~13 (the in-core implementation is
used when compiling with or without OpenSSL in those older versions),
could fail only under this circumstance.

The new cryptohash facilities can fail for reasons other than OOMs, like
attempting MD5 when FIPS is enabled (upstream OpenSSL allows that up to
1.0.2, Fedora and Photon patch OpenSSL 1.1.1 to allow that), so this
would cause incorrect reports to show up.

This commit extends the cryptohash APIs so as callers of those routines
can fetch more context when an error happens, by using a new routine
called pg_cryptohash_error().  The error states are stored within each
implementation's internal context data, so as it is possible to extend
the logic depending on what's suited for an implementation.  The default
implementation requires few error states, but OpenSSL could report
various issues depending on its internal state so more is needed in
cryptohash_openssl.c, and the code is shaped so as we are always able to
grab the necessary information.

The core code is changed to adapt to the new error routine, painting
more "const" across the call stack where the static errors are stored,
particularly in authentication code paths on variables that provide
log details.  This way, any future changes would warn if attempting to
free these strings.  The MD5 authentication code was also a bit blurry
about the handling of "logdetail" (LOG sent to the postmaster), so
improve the comments related that, while on it.

The origin of the problem is 87ae969, that introduced the centralized
cryptohash facility.  Extra changes are done for pgcrypto in v14 for the
non-OpenSSL code path to cope with the improvements done by this
commit.

Reported-by: Michael Mühlbeyer
Author: Michael Paquier
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/89B7F072-5BBE-4C92-903E-D83E865D9367@trivadis.com
Backpatch-through: 14

6 files changed, 63 insertions, 51 deletions

diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c
index 65bb7339589..a415100c3b1 100644
--- a/src/backend/commands/user.c
+++ b/src/backend/commands/user.c
@@ -393,7 +393,7 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt)
 	if (password)
 	{
 		char	   *shadow_pass;
-		char	   *logdetail;
+		const char *logdetail = NULL;
 
 		/*
 		 * Don't allow an empty password. Libpq treats an empty password the
@@ -835,7 +835,7 @@ AlterRole(AlterRoleStmt *stmt)
 	if (password)
 	{
 		char	   *shadow_pass;
-		char	   *logdetail;
+		const char *logdetail = NULL;
 
 		/* Like in CREATE USER, don't allow an empty password. */
 		if (password[0] == '\0' ||
diff --git a/src/backend/libpq/auth-scram.c b/src/backend/libpq/auth-scram.c
index f9e1026a12c..043134419c3 100644
--- a/src/backend/libpq/auth-scram.c
+++ b/src/backend/libpq/auth-scram.c
@@ -327,7 +327,7 @@ pg_be_scram_init(Port *port,
  */
 int
 pg_be_scram_exchange(void *opaq, const char *input, int inputlen,
-					 char **output, int *outputlen, char **logdetail)
+					 char **output, int *outputlen, const char **logdetail)
 {
 	scram_state *state = (scram_state *) opaq;
 	int			result;
diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c
index be14f2f16dd..0dd83cde066 100644
--- a/src/backend/libpq/auth.c
+++ b/src/backend/libpq/auth.c
@@ -47,7 +47,7 @@
  */
 static void sendAuthRequest(Port *port, AuthRequest areq, const char *extradata,
 							int extralen);
-static void auth_failed(Port *port, int status, char *logdetail);
+static void auth_failed(Port *port, int status, const char *logdetail);
 static char *recv_password_packet(Port *port);
 static void set_authn_id(Port *port, const char *id);
 
@@ -56,11 +56,11 @@ static void set_authn_id(Port *port, const char *id);
  * Password-based authentication methods (password, md5, and scram-sha-256)
  *----------------------------------------------------------------
  */
-static int	CheckPasswordAuth(Port *port, char **logdetail);
-static int	CheckPWChallengeAuth(Port *port, char **logdetail);
+static int	CheckPasswordAuth(Port *port, const char **logdetail);
+static int	CheckPWChallengeAuth(Port *port, const char **logdetail);
 
-static int	CheckMD5Auth(Port *port, char *shadow_pass, char **logdetail);
-static int	CheckSCRAMAuth(Port *port, char *shadow_pass, char **logdetail);
+static int	CheckMD5Auth(Port *port, char *shadow_pass, const char **logdetail);
+static int	CheckSCRAMAuth(Port *port, char *shadow_pass, const char **logdetail);
 
 
 /*----------------------------------------------------------------
@@ -258,7 +258,7 @@ ClientAuthentication_hook_type ClientAuthentication_hook = NULL;
  * particular, if logdetail isn't NULL, we send that string to the log.
  */
 static void
-auth_failed(Port *port, int status, char *logdetail)
+auth_failed(Port *port, int status, const char *logdetail)
 {
 	const char *errstr;
 	char	   *cdetail;
@@ -394,7 +394,7 @@ void
 ClientAuthentication(Port *port)
 {
 	int			status = STATUS_ERROR;
-	char	   *logdetail = NULL;
+	const char *logdetail = NULL;
 
 	/*
 	 * Get the authentication method to use for this frontend/database
@@ -780,7 +780,7 @@ recv_password_packet(Port *port)
  * Plaintext password authentication.
  */
 static int
-CheckPasswordAuth(Port *port, char **logdetail)
+CheckPasswordAuth(Port *port, const char **logdetail)
 {
 	char	   *passwd;
 	int			result;
@@ -815,7 +815,7 @@ CheckPasswordAuth(Port *port, char **logdetail)
  * MD5 and SCRAM authentication.
  */
 static int
-CheckPWChallengeAuth(Port *port, char **logdetail)
+CheckPWChallengeAuth(Port *port, const char **logdetail)
 {
 	int			auth_result;
 	char	   *shadow_pass;
@@ -875,7 +875,7 @@ CheckPWChallengeAuth(Port *port, char **logdetail)
 }
 
 static int
-CheckMD5Auth(Port *port, char *shadow_pass, char **logdetail)
+CheckMD5Auth(Port *port, char *shadow_pass, const char **logdetail)
 {
 	char		md5Salt[4];		/* Password salt */
 	char	   *passwd;
@@ -912,7 +912,7 @@ CheckMD5Auth(Port *port, char *shadow_pass, char **logdetail)
 }
 
 static int
-CheckSCRAMAuth(Port *port, char *shadow_pass, char **logdetail)
+CheckSCRAMAuth(Port *port, char *shadow_pass, const char **logdetail)
 {
 	StringInfoData sasl_mechs;
 	int			mtype;
@@ -3240,6 +3240,8 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 	md5trailer = packet->vector;
 	for (i = 0; i < encryptedpasswordlen; i += RADIUS_VECTOR_LENGTH)
 	{
+		const char *errstr = NULL;
+
 		memcpy(cryptvector + strlen(secret), md5trailer, RADIUS_VECTOR_LENGTH);
 
 		/*
@@ -3248,10 +3250,12 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		 */
 		md5trailer = encryptedpassword + i;
 
-		if (!pg_md5_binary(cryptvector, strlen(secret) + RADIUS_VECTOR_LENGTH, encryptedpassword + i))
+		if (!pg_md5_binary(cryptvector, strlen(secret) + RADIUS_VECTOR_LENGTH,
+						   encryptedpassword + i, &errstr))
 		{
 			ereport(LOG,
-					(errmsg("could not perform MD5 encryption of password")));
+					(errmsg("could not perform MD5 encryption of password: %s",
+							errstr)));
 			pfree(cryptvector);
 			pg_freeaddrinfo_all(hint.ai_family, serveraddrs);
 			return STATUS_ERROR;
@@ -3336,6 +3340,7 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 		struct timeval timeout;
 		struct timeval now;
 		int64		timeoutval;
+		const char *errstr = NULL;
 
 		gettimeofday(&now, NULL);
 		timeoutval = (endtime.tv_sec * 1000000 + endtime.tv_usec) - (now.tv_sec * 1000000 + now.tv_usec);
@@ -3454,10 +3459,11 @@ PerformRadiusTransaction(const char *server, const char *secret, const char *por
 
 		if (!pg_md5_binary(cryptvector,
 						   packetlength + strlen(secret),
-						   encryptedpassword))
+						   encryptedpassword, &errstr))
 		{
 			ereport(LOG,
-					(errmsg("could not perform MD5 encryption of received packet")));
+					(errmsg("could not perform MD5 encryption of received packet: %s",
+							errstr)));
 			pfree(cryptvector);
 			continue;
 		}
diff --git a/src/backend/libpq/crypt.c b/src/backend/libpq/crypt.c
index 3fcad991a7e..321603c1d8c 100644
--- a/src/backend/libpq/crypt.c
+++ b/src/backend/libpq/crypt.c
@@ -34,7 +34,7 @@
  * sent to the client, to avoid giving away user information!
  */
 char *
-get_role_password(const char *role, char **logdetail)
+get_role_password(const char *role, const char **logdetail)
 {
 	TimestampTz vuntil = 0;
 	HeapTuple	roleTup;
@@ -116,6 +116,7 @@ encrypt_password(PasswordType target_type, const char *role,
 {
 	PasswordType guessed_type = get_password_type(password);
 	char	   *encrypted_password;
+	const char *errstr = NULL;
 
 	if (guessed_type != PASSWORD_TYPE_PLAINTEXT)
 	{
@@ -132,8 +133,8 @@ encrypt_password(PasswordType target_type, const char *role,
 			encrypted_password = palloc(MD5_PASSWD_LEN + 1);
 
 			if (!pg_md5_encrypt(password, role, strlen(role),
-								encrypted_password))
-				elog(ERROR, "password encryption failed");
+								encrypted_password, &errstr))
+				elog(ERROR, "password encryption failed: %s", errstr);
 			return encrypted_password;
 
 		case PASSWORD_TYPE_SCRAM_SHA_256:
@@ -159,17 +160,18 @@ encrypt_password(PasswordType target_type, const char *role,
  * 'client_pass' is the response given by the remote user to the MD5 challenge.
  * 'md5_salt' is the salt used in the MD5 authentication challenge.
  *
- * In the error case, optionally store a palloc'd string at *logdetail
- * that will be sent to the postmaster log (but not the client).
+ * In the error case, save a string at *logdetail that will be sent to the
+ * postmaster log (but not the client).
  */
 int
 md5_crypt_verify(const char *role, const char *shadow_pass,
 				 const char *client_pass,
 				 const char *md5_salt, int md5_salt_len,
-				 char **logdetail)
+				 const char **logdetail)
 {
 	int			retval;
 	char		crypt_pwd[MD5_PASSWD_LEN + 1];
+	const char *errstr = NULL;
 
 	Assert(md5_salt_len > 0);
 
@@ -183,16 +185,13 @@ md5_crypt_verify(const char *role, const char *shadow_pass,
 
 	/*
 	 * Compute the correct answer for the MD5 challenge.
-	 *
-	 * We do not bother setting logdetail for any pg_md5_encrypt failure
-	 * below: the only possible error is out-of-memory, which is unlikely, and
-	 * if it did happen adding a psprintf call would only make things worse.
 	 */
 	/* stored password already encrypted, only do salt */
 	if (!pg_md5_encrypt(shadow_pass + strlen("md5"),
 						md5_salt, md5_salt_len,
-						crypt_pwd))
+						crypt_pwd, &errstr))
 	{
+		*logdetail = errstr;
 		return STATUS_ERROR;
 	}
 
@@ -215,15 +214,16 @@ md5_crypt_verify(const char *role, const char *shadow_pass,
  * pg_authid.rolpassword.
  * 'client_pass' is the password given by the remote user.
  *
- * In the error case, optionally store a palloc'd string at *logdetail
- * that will be sent to the postmaster log (but not the client).
+ * In the error case, store a string at *logdetail that will be sent to the
+ * postmaster log (but not the client).
  */
 int
 plain_crypt_verify(const char *role, const char *shadow_pass,
 				   const char *client_pass,
-				   char **logdetail)
+				   const char **logdetail)
 {
 	char		crypt_client_pass[MD5_PASSWD_LEN + 1];
+	const char *errstr = NULL;
 
 	/*
 	 * Client sent password in plaintext.  If we have an MD5 hash stored, hash
@@ -251,14 +251,10 @@ plain_crypt_verify(const char *role, const char *shadow_pass,
 			if (!pg_md5_encrypt(client_pass,
 								role,
 								strlen(role),
-								crypt_client_pass))
+								crypt_client_pass,
+								&errstr))
 			{
-				/*
-				 * We do not bother setting logdetail for pg_md5_encrypt
-				 * failure: the only possible error is out-of-memory, which is
-				 * unlikely, and if it did happen adding a psprintf call would
-				 * only make things worse.
-				 */
+				*logdetail = errstr;
 				return STATUS_ERROR;
 			}
 			if (strcmp(crypt_client_pass, shadow_pass) == 0)
diff --git a/src/backend/replication/backup_manifest.c b/src/backend/replication/backup_manifest.c
index 04ca455ace8..e6b91f5e2bd 100644
--- a/src/backend/replication/backup_manifest.c
+++ b/src/backend/replication/backup_manifest.c
@@ -67,7 +67,8 @@ InitializeBackupManifest(backup_manifest_info *manifest,
 		manifest->buffile = BufFileCreateTemp(false);
 		manifest->manifest_ctx = pg_cryptohash_create(PG_SHA256);
 		if (pg_cryptohash_init(manifest->manifest_ctx) < 0)
-			elog(ERROR, "failed to initialize checksum of backup manifest");
+			elog(ERROR, "failed to initialize checksum of backup manifest: %s",
+				 pg_cryptohash_error(manifest->manifest_ctx));
 	}
 
 	manifest->manifest_size = UINT64CONST(0);
@@ -334,7 +335,8 @@ SendBackupManifest(backup_manifest_info *manifest)
 	manifest->still_checksumming = false;
 	if (pg_cryptohash_final(manifest->manifest_ctx, checksumbuf,
 							sizeof(checksumbuf)) < 0)
-		elog(ERROR, "failed to finalize checksum of backup manifest");
+		elog(ERROR, "failed to finalize checksum of backup manifest: %s",
+			 pg_cryptohash_error(manifest->manifest_ctx));
 	AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
 
 	hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
@@ -401,7 +403,8 @@ AppendStringToManifest(backup_manifest_info *manifest, char *s)
 	if (manifest->still_checksumming)
 	{
 		if (pg_cryptohash_update(manifest->manifest_ctx, (uint8 *) s, len) < 0)
-			elog(ERROR, "failed to update checksum of backup manifest");
+			elog(ERROR, "failed to update checksum of backup manifest: %s",
+				 pg_cryptohash_error(manifest->manifest_ctx));
 	}
 	BufFileWrite(manifest->buffile, s, len);
 	manifest->manifest_size += len;
diff --git a/src/backend/utils/adt/cryptohashfuncs.c b/src/backend/utils/adt/cryptohashfuncs.c
index 6a0f0258e60..c977db2f0ea 100644
--- a/src/backend/utils/adt/cryptohashfuncs.c
+++ b/src/backend/utils/adt/cryptohashfuncs.c
@@ -35,15 +35,17 @@ md5_text(PG_FUNCTION_ARGS)
 	text	   *in_text = PG_GETARG_TEXT_PP(0);
 	size_t		len;
 	char		hexsum[MD5_HASH_LEN + 1];
+	const char *errstr = NULL;
 
 	/* Calculate the length of the buffer using varlena metadata */
 	len = VARSIZE_ANY_EXHDR(in_text);
 
 	/* get the hash result */
-	if (pg_md5_hash(VARDATA_ANY(in_text), len, hexsum) == false)
+	if (pg_md5_hash(VARDATA_ANY(in_text), len, hexsum, &errstr) == false)
 		ereport(ERROR,
-				(errcode(ERRCODE_OUT_OF_MEMORY),
-				 errmsg("out of memory")));
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 errmsg("could not compute %s hash: %s", "MD5",
+						errstr)));
 
 	/* convert to text and return it */
 	PG_RETURN_TEXT_P(cstring_to_text(hexsum));
@@ -58,12 +60,14 @@ md5_bytea(PG_FUNCTION_ARGS)
 	bytea	   *in = PG_GETARG_BYTEA_PP(0);
 	size_t		len;
 	char		hexsum[MD5_HASH_LEN + 1];
+	const char *errstr = NULL;
 
 	len = VARSIZE_ANY_EXHDR(in);
-	if (pg_md5_hash(VARDATA_ANY(in), len, hexsum) == false)
+	if (pg_md5_hash(VARDATA_ANY(in), len, hexsum, &errstr) == false)
 		ereport(ERROR,
-				(errcode(ERRCODE_OUT_OF_MEMORY),
-				 errmsg("out of memory")));
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 errmsg("could not compute %s hash: %s", "MD5",
+						errstr)));
 
 	PG_RETURN_TEXT_P(cstring_to_text(hexsum));
 }
@@ -111,12 +115,15 @@ cryptohash_internal(pg_cryptohash_type type, bytea *input)
 
 	ctx = pg_cryptohash_create(type);
 	if (pg_cryptohash_init(ctx) < 0)
-		elog(ERROR, "could not initialize %s context", typestr);
+		elog(ERROR, "could not initialize %s context: %s", typestr,
+			 pg_cryptohash_error(ctx));
 	if (pg_cryptohash_update(ctx, data, len) < 0)
-		elog(ERROR, "could not update %s context", typestr);
+		elog(ERROR, "could not update %s context: %s", typestr,
+			 pg_cryptohash_error(ctx));
 	if (pg_cryptohash_final(ctx, (unsigned char *) VARDATA(result),
 							digest_len) < 0)
-		elog(ERROR, "could not finalize %s context", typestr);
+		elog(ERROR, "could not finalize %s context: %s", typestr,
+			 pg_cryptohash_error(ctx));
 	pg_cryptohash_free(ctx);
 
 	SET_VARSIZE(result, digest_len + VARHDRSZ);