Clean up weird corner cases in lexing of psql meta-command arguments.

These changes allow backtick command evaluation and psql variable
interpolation to happen on substrings of a single meta-command argument.
Formerly, no such evaluations happened at all if the backtick or colon
wasn't the first character of the argument, and we considered an argument
completed as soon as we'd processed one backtick, variable reference, or
quoted substring.  A string like 'FOO'BAR was thus taken as two arguments
not one, not exactly what one would expect.  In the new coding, an argument
is considered terminated only by unquoted whitespace or backslash.

Also, clean up a bunch of omissions, infelicities and outright errors in
the psql documentation of variables and metacommand argument syntax.

3 files changed, 172 insertions, 179 deletions

diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 6d9cd6492f6..2c389021be7 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -121,7 +121,7 @@ HandleSlashCmds(PsqlScanState scan_state,
 		/* eat any remaining arguments after a valid command */
 		/* note we suppress evaluation of backticks here */
 		while ((arg = psql_scan_slash_option(scan_state,
-											 OT_VERBATIM, NULL, false)))
+											 OT_NO_EVAL, NULL, false)))
 		{
 			psql_error("\\%s: extra argument \"%s\" ignored\n", cmd, arg);
 			free(arg);
diff --git a/src/bin/psql/psqlscan.h b/src/bin/psql/psqlscan.h
index b2545d0ebf2..6264def953d 100644
--- a/src/bin/psql/psqlscan.h
+++ b/src/bin/psql/psqlscan.h
@@ -33,7 +33,7 @@ enum slash_option_type
 	OT_SQLIDHACK,				/* SQL identifier, but don't downcase */
 	OT_FILEPIPE,				/* it's a filename or pipe */
 	OT_WHOLE_LINE,				/* just snarf the rest of the line */
-	OT_VERBATIM					/* literal (no backticks or variables) */
+	OT_NO_EVAL					/* no expansion of backticks or variables */
 };
 
 
diff --git a/src/bin/psql/psqlscan.l b/src/bin/psql/psqlscan.l
index 1df8f3aa4f6..d4a9d94a437 100644
--- a/src/bin/psql/psqlscan.l
+++ b/src/bin/psql/psqlscan.l
@@ -103,6 +103,8 @@ static PQExpBuffer output_buf;	/* current output buffer */
 /* these variables do not need to be saved across calls */
 static enum slash_option_type option_type;
 static char *option_quote;
+static int	unquoted_option_chars;
+static int	backtick_start_offset;
 
 
 /* Return values from yylex() */
@@ -114,6 +116,7 @@ static char *option_quote;
 
 int	yylex(void);
 
+static void evaluate_backtick(void);
 static void push_new_buffer(const char *newstr, const char *varname);
 static void pop_buffer_stack(PsqlScanState state);
 static bool var_is_current_source(PsqlScanState state, const char *varname);
@@ -182,11 +185,11 @@ static void escape_variable(bool as_ident);
 %x xus
 /* Additional exclusive states for psql only: lex backslash commands */
 %x xslashcmd
+%x xslashargstart
 %x xslasharg
 %x xslashquote
 %x xslashbackquote
-%x xslashdefaultarg
-%x xslashquotedarg
+%x xslashdquote
 %x xslashwholeline
 %x xslashend
 
@@ -900,17 +903,53 @@ other			.
 
 }
 
-<xslasharg>{
-	/* eat any whitespace, then decide what to do at first nonblank */
+<xslashargstart>{
+	/*
+	 * Discard any whitespace before argument, then go to xslasharg state.
+	 * An exception is that "|" is only special at start of argument, so we
+	 * check for it here.
+	 */
 
 {space}+		{ }
 
-"\\"			{
+"|"				{
+					if (option_type == OT_FILEPIPE)
+					{
+						/* treat like whole-string case */
+						ECHO;
+						BEGIN(xslashwholeline);
+					}
+					else
+					{
+						/* vertical bar is not special otherwise */
+						yyless(0);
+						BEGIN(xslasharg);
+					}
+				}
+
+{other}			{
+					yyless(0);
+					BEGIN(xslasharg);
+				}
+
+}
+
+<xslasharg>{
+	/*
+	 * Default processing of text in a slash command's argument.
+	 *
+	 * Note: unquoted_option_chars counts the number of characters at the
+	 * end of the argument that were not subject to any form of quoting.
+	 * psql_scan_slash_option needs this to strip trailing semicolons safely.
+	 */
+
+{space}|"\\"	{
 					/*
+					 * Unquoted space is end of arg; do not eat.  Likewise
 					 * backslash is end of command or next command, do not eat
 					 *
 					 * XXX this means we can't conveniently accept options
-					 * that start with a backslash; therefore, option
+					 * that include unquoted backslashes; therefore, option
 					 * processing that encourages use of backslashes is rather
 					 * broken.
 					 */
@@ -920,26 +959,27 @@ other			.
 
 {quote}			{
 					*option_quote = '\'';
+					unquoted_option_chars = 0;
 					BEGIN(xslashquote);
 				}
 
 "`"				{
-					if (option_type == OT_VERBATIM)
-					{
-						/* in verbatim mode, backquote is not special */
-						ECHO;
-						BEGIN(xslashdefaultarg);
-					}
-					else
-					{
-						*option_quote = '`';
-						BEGIN(xslashbackquote);
-					}
+					backtick_start_offset = output_buf->len;
+					*option_quote = '`';
+					unquoted_option_chars = 0;
+					BEGIN(xslashbackquote);
+				}
+
+{dquote}		{
+					ECHO;
+					*option_quote = '"';
+					unquoted_option_chars = 0;
+					BEGIN(xslashdquote);
 				}
 
 :{variable_char}+	{
 					/* Possible psql variable substitution */
-					if (option_type == OT_VERBATIM)
+					if (option_type == OT_NO_EVAL)
 						ECHO;
 					else
 					{
@@ -959,71 +999,54 @@ other			.
 						 */
 						if (value)
 							appendPQExpBufferStr(output_buf, value);
-					}
-
-					*option_quote = ':';
+						else
+							ECHO;
 
-					return LEXRES_OK;
+						*option_quote = ':';
+					}
+					unquoted_option_chars = 0;
 				}
 
 :'{variable_char}+'	{
-					if (option_type == OT_VERBATIM)
+					if (option_type == OT_NO_EVAL)
 						ECHO;
 					else
 					{
 						escape_variable(false);
-						return LEXRES_OK;
+						*option_quote = ':';
 					}
+					unquoted_option_chars = 0;
 				}
 
 
 :\"{variable_char}+\"	{
-					if (option_type == OT_VERBATIM)
+					if (option_type == OT_NO_EVAL)
 						ECHO;
 					else
 					{
 						escape_variable(true);
-						return LEXRES_OK;
+						*option_quote = ':';
 					}
+					unquoted_option_chars = 0;
 				}
 
 :'{variable_char}*	{
 					/* Throw back everything but the colon */
 					yyless(1);
+					unquoted_option_chars++;
 					ECHO;
-					BEGIN(xslashdefaultarg);
 				}
 
 :\"{variable_char}*	{
 					/* Throw back everything but the colon */
 					yyless(1);
+					unquoted_option_chars++;
 					ECHO;
-					BEGIN(xslashdefaultarg);
-				}
-
-"|"				{
-					ECHO;
-					if (option_type == OT_FILEPIPE)
-					{
-						/* treat like whole-string case */
-						BEGIN(xslashwholeline);
-					}
-					else
-					{
-						/* treat like default case */
-						BEGIN(xslashdefaultarg);
-					}
-				}
-
-{dquote}		{
-					*option_quote = '"';
-					ECHO;
-					BEGIN(xslashquotedarg);
 				}
 
 {other}			{
+					unquoted_option_chars++;
 					ECHO;
-					BEGIN(xslashdefaultarg);
 				}
 
 }
@@ -1034,7 +1057,7 @@ other			.
 	 * sequences
 	 */
 
-{quote}			{ return LEXRES_OK; }
+{quote}			{ BEGIN(xslasharg); }
 
 {xqdouble}		{ appendPQExpBufferChar(output_buf, '\''); }
 
@@ -1064,55 +1087,28 @@ other			.
 
 <xslashbackquote>{
 	/*
-	 * backticked text: copy everything until next backquote or end of line.
-	 * Invocation of the command will happen in psql_scan_slash_option.
+	 * backticked text: copy everything until next backquote, then evaluate.
+	 *
+	 * XXX Possible future behavioral change: substitute for :VARIABLE?
 	 */
 
-"`"				{ return LEXRES_OK; }
-
-{other}|\n		{ ECHO; }
-
-}
-
-<xslashdefaultarg>{
-	/*
-	 * Copy everything until unquoted whitespace or end of line.  Quotes
-	 * do not get stripped yet.
-	 */
-
-{space}			{
-					yyless(0);
-					return LEXRES_OK;
-				}
-
-"\\"			{
-					/*
-					 * unquoted backslash is end of command or next command,
-					 * do not eat
-					 *
-					 * (this was not the behavior pre-8.0, but it seems
-					 * consistent)
-					 */
-					yyless(0);
-					return LEXRES_OK;
-				}
-
-{dquote}		{
-					*option_quote = '"';
-					ECHO;
-					BEGIN(xslashquotedarg);
+"`"				{
+					/* In NO_EVAL mode, don't evaluate the command */
+					if (option_type != OT_NO_EVAL)
+						evaluate_backtick();
+					BEGIN(xslasharg);
 				}
 
-{other}			{ ECHO; }
+{other}|\n		{ ECHO; }
 
 }
 
-<xslashquotedarg>{
-	/* double-quoted text within a default-type argument: copy */
+<xslashdquote>{
+	/* double-quoted text: copy verbatim, including the double quotes */
 
 {dquote}		{
 					ECHO;
-					BEGIN(xslashdefaultarg);
+					BEGIN(xslasharg);
 				}
 
 {other}|\n		{ ECHO; }
@@ -1461,7 +1457,7 @@ psql_scan_slash_command(PsqlScanState state)
  * letters.
  *
  * if quote is not NULL, *quote is set to 0 if no quoting was found, else
- * the quote symbol.
+ * the last quote symbol used in the argument.
  *
  * if semicolon is true, unquoted trailing semicolon(s) that would otherwise
  * be taken as part of the option string will be stripped.
@@ -1480,7 +1476,6 @@ psql_scan_slash_option(PsqlScanState state,
 	PQExpBufferData mybuf;
 	int			lexresult;
 	char		local_quote;
-	bool		badarg;
 
 	/* Must be scanning already */
 	psql_assert(state->scanbufhandle);
@@ -1497,6 +1492,7 @@ psql_scan_slash_option(PsqlScanState state,
 	output_buf = &mybuf;
 	option_type = type;
 	option_quote = quote;
+	unquoted_option_chars = 0;
 
 	if (state->buffer_stack != NULL)
 		yy_switch_to_buffer(state->buffer_stack->buf);
@@ -1506,7 +1502,7 @@ psql_scan_slash_option(PsqlScanState state,
 	if (type == OT_WHOLE_LINE)
 		BEGIN(xslashwholeline);
 	else
-		BEGIN(xslasharg);
+		BEGIN(xslashargstart);
 
 	/* And lex. */
 	lexresult = yylex();
@@ -1517,85 +1513,18 @@ psql_scan_slash_option(PsqlScanState state,
 	 * a quoted string, as indicated by YY_START, EOL is an error.
 	 */
 	psql_assert(lexresult == LEXRES_EOL || lexresult == LEXRES_OK);
-	badarg = false;
+
 	switch (YY_START)
 	{
-		case xslasharg:
-			/* empty arg, or possibly a psql variable substitution */
-			break;
-		case xslashquote:
-			if (lexresult != LEXRES_OK)
-				badarg = true;		/* hit EOL not ending quote */
-			break;
-		case xslashbackquote:
-			if (lexresult != LEXRES_OK)
-				badarg = true;		/* hit EOL not ending quote */
-			else
-			{
-				/* Perform evaluation of backticked command */
-				char	   *cmd = mybuf.data;
-				FILE	   *fd;
-				bool		error = false;
-				PQExpBufferData output;
-				char		buf[512];
-				size_t		result;
-
-				fd = popen(cmd, PG_BINARY_R);
-				if (!fd)
-				{
-					psql_error("%s: %s\n", cmd, strerror(errno));
-					error = true;
-				}
-
-				initPQExpBuffer(&output);
-
-				if (!error)
-				{
-					do
-					{
-						result = fread(buf, 1, sizeof(buf), fd);
-						if (ferror(fd))
-						{
-							psql_error("%s: %s\n", cmd, strerror(errno));
-							error = true;
-							break;
-						}
-						appendBinaryPQExpBuffer(&output, buf, result);
-					} while (!feof(fd));
-				}
-
-				if (fd && pclose(fd) == -1)
-				{
-					psql_error("%s: %s\n", cmd, strerror(errno));
-					error = true;
-				}
-
-				if (PQExpBufferBroken(&output))
-				{
-					psql_error("%s: out of memory\n", cmd);
-					error = true;
-				}
-
-				/* Now done with cmd, transfer result to mybuf */
-				resetPQExpBuffer(&mybuf);
-
-				if (!error)
-				{
-					/* strip any trailing newline */
-					if (output.len > 0 &&
-						output.data[output.len - 1] == '\n')
-						output.len--;
-					appendBinaryPQExpBuffer(&mybuf, output.data, output.len);
-				}
-
-				termPQExpBuffer(&output);
-			}
+		case xslashargstart:
+			/* empty arg */
 			break;
-		case xslashdefaultarg:
-			/* Strip any trailing semi-colons if requested */
+		case xslasharg:
+			/* Strip any unquoted trailing semi-colons if requested */
 			if (semicolon)
 			{
-				while (mybuf.len > 0 &&
+				while (unquoted_option_chars-- > 0 &&
+					   mybuf.len > 0 &&
 					   mybuf.data[mybuf.len - 1] == ';')
 				{
 					mybuf.data[--mybuf.len] = '\0';
@@ -1642,10 +1571,13 @@ psql_scan_slash_option(PsqlScanState state,
 				}
 			}
 			break;
-		case xslashquotedarg:
-			/* must have hit EOL inside double quotes */
-			badarg = true;
-			break;
+		case xslashquote:
+		case xslashbackquote:
+		case xslashdquote:
+			/* must have hit EOL inside quotes */
+			psql_error("unterminated quoted string\n");
+			termPQExpBuffer(&mybuf);
+			return NULL;
 		case xslashwholeline:
 			/* always okay */
 			break;
@@ -1655,13 +1587,6 @@ psql_scan_slash_option(PsqlScanState state,
 			exit(1);
 	}
 
-	if (badarg)
-	{
-		psql_error("unterminated quoted string\n");
-		termPQExpBuffer(&mybuf);
-		return NULL;
-	}
-
 	/*
 	 * An unquoted empty argument isn't possible unless we are at end of
 	 * command.  Return NULL instead.
@@ -1702,6 +1627,74 @@ psql_scan_slash_command_end(PsqlScanState state)
 	/* There are no possible errors in this lex state... */
 }
 
+/*
+ * Evaluate a backticked substring of a slash command's argument.
+ *
+ * The portion of output_buf starting at backtick_start_offset is evaluated
+ * as a shell command and then replaced by the command's output.
+ */
+static void
+evaluate_backtick(void)
+{
+	char	   *cmd = output_buf->data + backtick_start_offset;
+	PQExpBufferData cmd_output;
+	FILE	   *fd;
+	bool		error = false;
+	char		buf[512];
+	size_t		result;
+
+	initPQExpBuffer(&cmd_output);
+
+	fd = popen(cmd, PG_BINARY_R);
+	if (!fd)
+	{
+		psql_error("%s: %s\n", cmd, strerror(errno));
+		error = true;
+	}
+
+	if (!error)
+	{
+		do
+		{
+			result = fread(buf, 1, sizeof(buf), fd);
+			if (ferror(fd))
+			{
+				psql_error("%s: %s\n", cmd, strerror(errno));
+				error = true;
+				break;
+			}
+			appendBinaryPQExpBuffer(&cmd_output, buf, result);
+		} while (!feof(fd));
+	}
+
+	if (fd && pclose(fd) == -1)
+	{
+		psql_error("%s: %s\n", cmd, strerror(errno));
+		error = true;
+	}
+
+	if (PQExpBufferBroken(&cmd_output))
+	{
+		psql_error("%s: out of memory\n", cmd);
+		error = true;
+	}
+
+	/* Now done with cmd, delete it from output_buf */
+	output_buf->len = backtick_start_offset;
+	output_buf->data[output_buf->len] = '\0';
+
+	/* If no error, transfer result to output_buf */
+	if (!error)
+	{
+		/* strip any trailing newline */
+		if (cmd_output.len > 0 &&
+			cmd_output.data[cmd_output.len - 1] == '\n')
+			cmd_output.len--;
+		appendBinaryPQExpBuffer(output_buf, cmd_output.data, cmd_output.len);
+	}
+
+	termPQExpBuffer(&cmd_output);
+}
 
 /*
  * Push the given string onto the stack of stuff to scan.