Implement a solution to the 'Turkish locale downcases I incorrectly'

problem, per previous discussion.  Make some additional changes to
centralize the knowledge of just how identifier downcasing is done,
in hopes of simplifying any future tweaking in this area.

3 files changed, 87 insertions, 45 deletions

diff --git a/src/backend/parser/keywords.c b/src/backend/parser/keywords.c
index c4048b4c1d8..f4f454715c6 100644
--- a/src/backend/parser/keywords.c
+++ b/src/backend/parser/keywords.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/parser/keywords.c,v 1.141 2003/08/04 02:40:01 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/parser/keywords.c,v 1.141.4.1 2004/02/21 00:35:13 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -365,17 +365,13 @@ ScanKeywordLookup(const char *text)
 
 	/*
 	 * Apply an ASCII-only downcasing.	We must not use tolower() since it
-	 * may produce the wrong translation in some locales (eg, Turkish),
-	 * and we don't trust isupper() very much either.  In an ASCII-based
-	 * encoding the tests against A and Z are sufficient, but we also
-	 * check isupper() so that we will work correctly under EBCDIC.  The
-	 * actual case conversion step should work for either ASCII or EBCDIC.
+	 * may produce the wrong translation in some locales (eg, Turkish).
 	 */
 	for (i = 0; i < len; i++)
 	{
 		char		ch = text[i];
 
-		if (ch >= 'A' && ch <= 'Z' && isupper((unsigned char) ch))
+		if (ch >= 'A' && ch <= 'Z')
 			ch += 'a' - 'A';
 		word[i] = ch;
 	}
diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
index b10d4531851..c3a423a7f4a 100644
--- a/src/backend/parser/scan.l
+++ b/src/backend/parser/scan.l
@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/parser/scan.l,v 1.111 2003/10/09 19:13:23 petere Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/parser/scan.l,v 1.111.2.1 2004/02/21 00:35:13 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -26,6 +26,7 @@
 #include "parser/keywords.h"
 /* Not needed now that this file is compiled as part of gram.y */
 /* #include "parser/parse.h" */
+#include "parser/scansup.h"
 #include "utils/builtins.h"
 #include "mb/pg_wchar.h"
 
@@ -394,23 +395,15 @@ other			.
 					startlit();
 				}
 <xd>{xdstop}	{
+					char		   *ident;
+
 					BEGIN(INITIAL);
 					if (literallen == 0)
 						yyerror("zero-length delimited identifier");
+					ident = litbufdup();
 					if (literallen >= NAMEDATALEN)
-					{
-						int len;
-
-						len = pg_mbcliplen(literalbuf, literallen,
-										   NAMEDATALEN-1);
-						ereport(NOTICE,
-								(errcode(ERRCODE_NAME_TOO_LONG),
-								 errmsg("identifier \"%s\" will be truncated to \"%.*s\"",
-										literalbuf, len, literalbuf)));
-						literalbuf[len] = '\0';
-						literallen = len;
-					}
-					yylval.str = litbufdup();
+						truncate_identifier(ident, literallen, true);
+					yylval.str = ident;
 					return IDENT;
 				}
 <xd>{xddouble} {
@@ -532,7 +525,6 @@ other			.
 {identifier}	{
 					const ScanKeyword *keyword;
 					char		   *ident;
-					int				i;
 
 					/* Is it a keyword? */
 					keyword = ScanKeywordLookup(yytext);
@@ -545,28 +537,8 @@ other			.
 					/*
 					 * No.  Convert the identifier to lower case, and truncate
 					 * if necessary.
-					 *
-					 * Note: here we use a locale-dependent case conversion,
-					 * which seems appropriate under standard SQL rules, whereas
-					 * the keyword comparison was NOT locale-dependent.
 					 */
-					ident = pstrdup(yytext);
-					for (i = 0; ident[i]; i++)
-					{
-						if (isupper((unsigned char) ident[i]))
-							ident[i] = tolower((unsigned char) ident[i]);
-					}
-					if (i >= NAMEDATALEN)
-                    {
-						int len;
-
-						len = pg_mbcliplen(ident, i, NAMEDATALEN-1);
-						ereport(NOTICE,
-								(errcode(ERRCODE_NAME_TOO_LONG),
-								 errmsg("identifier \"%s\" will be truncated to \"%.*s\"",
-										ident, len, ident)));
-						ident[len] = '\0';
-                    }
+					ident = downcase_truncate_identifier(yytext, yyleng, true);
 					yylval.str = ident;
 					return IDENT;
 				}
diff --git a/src/backend/parser/scansup.c b/src/backend/parser/scansup.c
index e00d284edcc..14467dcdc77 100644
--- a/src/backend/parser/scansup.c
+++ b/src/backend/parser/scansup.c
@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/parser/scansup.c,v 1.24 2003/08/04 02:40:02 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/parser/scansup.c,v 1.24.4.1 2004/02/21 00:35:13 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -19,6 +19,8 @@
 
 #include "miscadmin.h"
 #include "parser/scansup.h"
+#include "mb/pg_wchar.h"
+
 
 /* ----------------
  *		scanstr
@@ -32,7 +34,7 @@
  */
 
 char *
-scanstr(char *s)
+scanstr(const char *s)
 {
 	char	   *newStr;
 	int			len,
@@ -109,3 +111,75 @@ scanstr(char *s)
 	newStr[j] = '\0';
 	return newStr;
 }
+
+
+/*
+ * downcase_truncate_identifier() --- do appropriate downcasing and
+ * truncation of an unquoted identifier.  Optionally warn of truncation.
+ *
+ * Returns a palloc'd string containing the adjusted identifier.
+ *
+ * Note: in some usages the passed string is not null-terminated.
+ *
+ * Note: the API of this function is designed to allow for downcasing
+ * transformations that increase the string length, but we don't yet
+ * support that.  If you want to implement it, you'll need to fix
+ * SplitIdentifierString() in utils/adt/varlena.c.
+ */
+char *
+downcase_truncate_identifier(const char *ident, int len, bool warn)
+{
+	char	   *result;
+	int			i;
+
+	result = palloc(len + 1);
+	/*
+	 * SQL99 specifies Unicode-aware case normalization, which we don't yet
+	 * have the infrastructure for.  Instead we use tolower() to provide a
+	 * locale-aware translation.  However, there are some locales where this
+	 * is not right either (eg, Turkish may do strange things with 'i' and
+	 * 'I').  Our current compromise is to use tolower() for characters with
+	 * the high bit set, and use an ASCII-only downcasing for 7-bit
+	 * characters.
+	 */
+	for (i = 0; i < len; i++)
+	{
+		unsigned char	ch = (unsigned char) ident[i];
+
+		if (ch >= 'A' && ch <= 'Z')
+			ch += 'a' - 'A';
+		else if (ch >= 0x80 && isupper(ch))
+			ch = tolower(ch);
+		result[i] = (char) ch;
+	}
+	result[i] = '\0';
+
+	if (i >= NAMEDATALEN)
+		truncate_identifier(result, i, warn);
+
+	return result;
+}
+
+/*
+ * truncate_identifier() --- truncate an identifier to NAMEDATALEN-1 bytes.
+ *
+ * The given string is modified in-place, if necessary.  A warning is
+ * issued if requested.
+ *
+ * We require the caller to pass in the string length since this saves a
+ * strlen() call in some common usages.
+ */
+void
+truncate_identifier(char *ident, int len, bool warn)
+{
+	if (len >= NAMEDATALEN)
+	{
+		len = pg_mbcliplen(ident, len, NAMEDATALEN-1);
+		if (warn)
+			ereport(NOTICE,
+					(errcode(ERRCODE_NAME_TOO_LONG),
+					 errmsg("identifier \"%s\" will be truncated to \"%.*s\"",
+							ident, len, ident)));
+		ident[len] = '\0';
+	}
+}