Use correct output device for Windows prompts.

This ensures that mapping of non-ascii prompts
to the correct code page occurs.

Bug report and original patch from Alexander Law,
reviewed and reworked by Noah Misch.

Backpatch to all live branches.

3 files changed, 41 insertions, 6 deletions

diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 786f4139b53..4c22f72a712 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -912,6 +912,17 @@ exec_command(const char *cmd,
 		char	   *fname = psql_scan_slash_option(scan_state,
 												   OT_NORMAL, NULL, true);
 
+#if defined(WIN32) && !defined(__CYGWIN__)
+
+		/*
+		 * XXX This does not work for all terminal environments or for output
+		 * containing non-ASCII characters; see comments in simple_prompt().
+		 */
+#define DEVTTY	"con"
+#else
+#define DEVTTY	"/dev/tty"
+#endif
+
 		expand_tilde(&fname);
 		/* This scrolls off the screen when using /dev/tty */
 		success = saveHistory(fname ? fname : DEVTTY, false);
diff --git a/src/include/port.h b/src/include/port.h
index f33ee15476c..ece64db7f70 100644
--- a/src/include/port.h
+++ b/src/include/port.h
@@ -93,11 +93,8 @@ extern BOOL AddUserToTokenDacl(HANDLE hToken);
 
 #if defined(WIN32) && !defined(__CYGWIN__)
 #define DEVNULL "nul"
-/* "con" does not work from the Msys 1.0.10 console (part of MinGW). */
-#define DEVTTY	"con"
 #else
 #define DEVNULL "/dev/null"
-#define DEVTTY "/dev/tty"
 #endif
 
 /*
diff --git a/src/port/sprompt.c b/src/port/sprompt.c
index b06daa2e995..f0eb77f483d 100644
--- a/src/port/sprompt.c
+++ b/src/port/sprompt.c
@@ -56,15 +56,42 @@ simple_prompt(const char *prompt, int maxlen, bool echo)
 	if (!destination)
 		return NULL;
 
+#ifdef WIN32
+
+	/*
+	 * A Windows console has an "input code page" and an "output code page";
+	 * these usually match each other, but they rarely match the "Windows ANSI
+	 * code page" defined at system boot and expected of "char *" arguments to
+	 * Windows API functions.  The Microsoft CRT write() implementation
+	 * automatically converts text between these code pages when writing to a
+	 * console.  To identify such file descriptors, it calls GetConsoleMode()
+	 * on the underlying HANDLE, which in turn requires GENERIC_READ access on
+	 * the HANDLE.  Opening termout in mode "w+" allows that detection to
+	 * succeed.  Otherwise, write() would not recognize the descriptor as a
+	 * console, and non-ASCII characters would display incorrectly.
+	 *
+	 * XXX fgets() still receives text in the console's input code page.  This
+	 * makes non-ASCII credentials unportable.
+	 */
+	termin = fopen("CONIN$", "r");
+	termout = fopen("CONOUT$", "w+");
+#else
+
 	/*
 	 * Do not try to collapse these into one "w+" mode file. Doesn't work on
 	 * some platforms (eg, HPUX 10.20).
 	 */
-	termin = fopen(DEVTTY, "r");
-	termout = fopen(DEVTTY, "w");
+	termin = fopen("/dev/tty", "r");
+	termout = fopen("/dev/tty", "w");
+#endif
 	if (!termin || !termout
 #ifdef WIN32
-	/* See DEVTTY comment for msys */
+	/*
+	 * Direct console I/O does not work from the MSYS 1.0.10 console.  Writes
+	 * reach nowhere user-visible; reads block indefinitely.  XXX This affects
+	 * most Windows terminal environments, including rxvt, mintty, Cygwin
+	 * xterm, Cygwin sshd, and PowerShell ISE.  Switch to a more-generic test.
+	 */
 		|| (getenv("OSTYPE") && strcmp(getenv("OSTYPE"), "msys") == 0)
 #endif
 		)