I really hope that I haven't missed anything in this one...

From: t-ishii@sra.co.jp

Attached are patches to enhance the multi-byte support.  (patches are
against 7/18 snapshot)

* determine encoding at initdb/createdb rather than compile time

Now initdb/createdb has an option to specify the encoding. Also, I
modified the syntax of CREATE DATABASE to accept encoding option. See
README.mb for more details.

For this purpose I have added new column "encoding" to pg_database.
Also pg_attribute and pg_class are changed to catch up the
modification to pg_database.  Actually I haved added pg_database_mb.h,
pg_attribute_mb.h and pg_class_mb.h. These are used only when MB is
enabled. The reason having separate files is I couldn't find a way to
use ifdef or whatever in those files. I have to admit it looks
ugly. No way.

* support for PGCLIENTENCODING when issuing COPY command

commands/copy.c modified.

* support for SQL92 syntax "SET NAMES"

See gram.y.

* support for LATIN2-5
* add UNICODE regression test case
* new test suite for MB

New directory test/mb added.

* clean up source files

Basic idea is to have MB's own subdirectory for easier maintenance.
These are include/mb and backend/utils/mb.

5 files changed, 1 insertions, 570 deletions

diff --git a/src/backend/regex/Makefile b/src/backend/regex/Makefile
index 42f635aaeb4..1d1a9ac4f68 100644
--- a/src/backend/regex/Makefile
+++ b/src/backend/regex/Makefile
@@ -4,7 +4,7 @@
 #    Makefile for regex
 #
 # IDENTIFICATION
-#    $Header: /cvsroot/pgsql/src/backend/regex/Makefile,v 1.6 1998/04/06 00:24:39 momjian Exp $
+#    $Header: /cvsroot/pgsql/src/backend/regex/Makefile,v 1.7 1998/07/24 03:31:24 scrappy Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -16,7 +16,6 @@ CFLAGS += -DPOSIX_MISTAKE
 
 OBJS = regcomp.o regerror.o regexec.o regfree.o
 ifdef MB
-OBJS += utils.o wstrcmp.o wstrncmp.o
 CFLAGS += -DMB=$(MB)
 endif
 
diff --git a/src/backend/regex/utftest.c b/src/backend/regex/utftest.c
deleted file mode 100644
index 28baf7255ef..00000000000
--- a/src/backend/regex/utftest.c
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * testing of utf2wchar()
- * $Id: utftest.c,v 1.1 1998/03/15 07:38:37 scrappy Exp $
- */
-#include <regex/regex.h>
-#include <regex/utils.h>
-#include <regex/regex2.h>
-
-#include <regex/pg_wchar.h>
-
-main()
-{
-  /* Example 1 from RFC2044 */
-  char utf1[] = {0x41,0xe2,0x89,0xa2,0xce,0x91,0x2e,0};
-  /* Example 2 from RFC2044 */
-  char utf2[] = {0x48,0x69,0x20,0x4d,0x6f,0x6d,0x20,0xe2,0x98,0xba,0x21,0};
-  /* Example 3 from RFC2044 */
-  char utf3[] = {0xe6,0x97,0xa5,0xe6,0x9c,0xac,0xe8,0xaa,0x9e,0};
-  char *utf[] = {utf1,utf2,utf3};
-  pg_wchar ucs[128];
-  pg_wchar *p;
-  int i;
-
-  for (i=0;i<sizeof(utf)/sizeof(char *);i++) {
-    pg_utf2wchar(utf[i],ucs);
-    p = ucs;
-    while(*p) {
-      printf("%04x ",*p);
-      p++;
-    }
-    printf("\n");
-  }
-}
diff --git a/src/backend/regex/utils.c b/src/backend/regex/utils.c
deleted file mode 100644
index 0f8844f41e3..00000000000
--- a/src/backend/regex/utils.c
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * misc conversion functions between pg_wchar and other encodings.
- * Tatsuo Ishii
- * $Id: utils.c,v 1.4 1998/07/18 18:34:08 momjian Exp $
- */
-#include <regex/pg_wchar.h>
-
-/*
- * conversion to pg_wchar is done by "table driven."
- * to add an encoding support, define mb2wchar_with_len(), mblen()
- * for the particular encoding. Note that if the encoding is only
- * supported in the client, you don't need to define 
- * mb2wchar_with_len() function (SJIS is the case).
- */
-typedef struct {
-  void	(*mb2wchar_with_len)();	/* convert a multi-byte string to a wchar */
-  int	(*mblen)();		/* returns the length of a multi-byte word */
-} pg_wchar_tbl;
-
-static void pg_euc2wchar_with_len
-(const unsigned char *from, pg_wchar *to, int len)
-{
-  while (*from && len > 0) {
-    if (*from == SS2) {
-      from++;
-      len--;
-      *to = 0xff & *from++;
-      len--;
-    } else if (*from == SS3) {
-      from++;
-      *to = *from++ << 8;
-      *to |= 0x3f & *from++;
-      len -= 3;
-    } else if (*from & 0x80) {
-      *to = *from++ << 8;
-      *to |= *from++;
-      len -= 2;
-    } else {
-      *to = *from++;
-      len--;
-    }
-    to++;
-  }
-  *to = 0;
-}
-
-static int pg_euc_mblen(const unsigned char *s)
-{
-  int len;
-
-  if (*s == SS2) {
-    len = 2;
-  } else if (*s == SS3) {
-    len = 3;
-  } else if (*s & 0x80) {
-    len = 2;
-  } else {
-    len = 1;
-  }
-  return(len);
-}
-
-/*
- * EUC_JP
- */
-static void pg_eucjp2wchar_with_len
-(const unsigned char *from, pg_wchar *to, int len)
-{
-  pg_euc2wchar_with_len(from,to,len);
-}
-
-static int pg_eucjp_mblen(const unsigned char *s)
-{
-  return(pg_euc_mblen(s));
-}
-
-/*
- * EUC_KR
- */
-static void pg_euckr2wchar_with_len
-(const unsigned char *from, pg_wchar *to, int len)
-{
-  pg_euc2wchar_with_len(from,to,len);
-}
-
-static int pg_euckr_mblen(const unsigned char *s)
-{
-  return(pg_euc_mblen(s));
-}
-
-/*
- * EUC_CN
- */
-static void pg_euccn2wchar_with_len
-(const unsigned char *from, pg_wchar *to, int len)
-{
-  while (*from && len > 0) {
-    if (*from == SS2) {
-      from++;
-      len--;
-      *to = 0x3f00 & (*from++ << 8);
-      *to = *from++;
-      len -= 2;
-    } else if (*from == SS3) {
-      from++;
-      *to = *from++ << 8;
-      *to |= 0x3f & *from++;
-      len -= 3;
-    } else if (*from & 0x80) {
-      *to = *from++ << 8;
-      *to |= *from++;
-      len -= 2;
-    } else {
-      *to = *from++;
-      len--;
-    }
-    to++;
-  }
-  *to = 0;
-}
-
-static int pg_euccn_mblen(const unsigned char *s)
-{
-  int len;
-
-  if (*s == SS2) {
-    len = 3;
-  } else if (*s == SS3) {
-    len = 3;
-  } else if (*s & 0x80) {
-    len = 2;
-  } else {
-    len = 1;
-  }
-  return(len);
-}
-
-/*
- * EUC_TW
- */
-static void pg_euctw2wchar_with_len
-(const unsigned char *from, pg_wchar *to, int len)
-{
-  while (*from && len > 0) {
-    if (*from == SS2) {
-      from++;
-      len--;
-      *to = *from++ << 16;
-      *to |= *from++ << 8;
-      *to |= *from++;
-      len -= 3;
-    } else if (*from == SS3) {
-      from++;
-      *to = *from++ << 8;
-      *to |= 0x3f & *from++;
-      len -= 3;
-    } else if (*from & 0x80) {
-      *to = *from++ << 8;
-      *to |= *from++;
-      len -= 2;
-    } else {
-      *to = *from++;
-      len--;
-    }
-    to++;
-  }
-  *to = 0;
-}
-
-static int pg_euctw_mblen(const unsigned char *s)
-{
-  int len;
-
-  if (*s == SS2) {
-    len = 4;
-  } else if (*s == SS3) {
-    len = 3;
-  } else if (*s & 0x80) {
-    len = 2;
-  } else {
-    len = 1;
-  }
-  return(len);
-}
-
-/*
- * convert UTF-8 to pg_wchar (UCS-2)
- * caller should allocate enough space for "to"
- * len: length of from.
- * "from" not necessarily null terminated.
- */
-static void pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
-{
-  unsigned char c1,c2,c3;
-  while (*from && len > 0) {
-    if ((*from & 0x80) == 0) {
-      *to = *from++;
-      len--;
-    } else if ((*from & 0xe0) == 0xc0) {
-      c1 = *from++ & 0x1f;
-      c2 = *from++ & 0x3f;
-      len -= 2;
-      *to = c1 << 6;
-      *to |= c2;
-    } else if ((*from & 0xe0) == 0xe0) {
-      c1 = *from++ & 0x0f;
-      c2 = *from++ & 0x3f;
-      c3 = *from++ & 0x3f;
-      len -= 3;
-      *to = c1 << 12;
-      *to |= c2 << 6;
-      *to |= c3;
-    }
-    to++;
-  }
-  *to = 0;
-}
-
-static int pg_utf_mblen(const unsigned char *s)
-{
-  int len = 1;
-
-  if ((*s & 0x80) == 0) {
-    len = 1;
-  } else if ((*s & 0xe0) == 0xc0) {
-    len = 2;
-  } else if ((*s & 0xe0) == 0xe0) {
-    len = 3;
-  }
-  return(len);
-}
-
-/*
- * convert mule internal code to pg_wchar
- * caller should allocate enough space for "to"
- * len: length of from.
- * "from" not necessarily null terminated.
- */
-static void pg_mule2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
-{
-  while (*from && len > 0) {
-    if (IS_LC1(*from)) {
-      *to = *from++ << 16;
-      *to |= *from++;
-      len -= 2;
-    } else if (IS_LCPRV1(*from)) {
-      from++;
-      *to = *from++ << 16;
-      *to |= *from++;
-      len -= 3;
-    } else if (IS_LC2(*from)) {
-      *to = *from++ << 16;
-      *to |= *from++ << 8;
-      *to |= *from++;
-      len -= 3;
-    } else if (IS_LCPRV2(*from)) {
-      from++;
-      *to = *from++ << 16;
-      *to |= *from++ << 8;
-      *to |= *from++;
-      len -= 4;
-    } else {	/* assume ASCII */
-      *to = (unsigned char)*from++;
-      len--;
-    }
-    to++;
-  }
-  *to = 0;
-}
-
-static int pg_mule_mblen(const unsigned char *s)
-{
-  int len;
-
-  if (IS_LC1(*s)) {
-    len = 2;
-  } else if (IS_LCPRV1(*s)) {
-    len = 3;
-  } else if (IS_LC2(*s)) {
-    len = 3;
-  } else if (IS_LCPRV2(*s)) {
-    len = 4;
-  } else {	/* assume ASCII */
-    len = 1;
-  }
-  return(len);
-}
-
-/*
- * ISO8859-1
- */
-static void pg_latin12wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
-{
-  while (*from && len-- > 0) {
-    *to++ = *from++;
-  }
-  *to = 0;
-}
-
-static int pg_latin1_mblen(const unsigned char *s)
-{
-  return(1);
-}
-
-/*
- * SJIS
- */
-static int pg_sjis_mblen(const unsigned char *s)
-{
-  int len;
-
-  if (*s >= 0xa1 && *s <= 0xdf) {	/* 1 byte kana? */
-    len = 1;
-  } else if (*s > 0x7f) {	/* kanji? */
-    len = 2;
-  } else {	/* should be ASCII */
-    len = 1;
-  }
-  return(len);
-}
-
-static pg_wchar_tbl pg_wchar_table[] = {
-  {pg_eucjp2wchar_with_len, pg_eucjp_mblen},
-  {pg_euccn2wchar_with_len, pg_euccn_mblen},
-  {pg_euckr2wchar_with_len, pg_euckr_mblen},
-  {pg_euctw2wchar_with_len, pg_euctw_mblen},
-  {pg_utf2wchar_with_len, pg_utf_mblen},
-  {pg_mule2wchar_with_len, pg_mule_mblen},
-  {pg_latin12wchar_with_len, pg_latin1_mblen},
-  {0, 0},
-  {0, 0},
-  {0, 0},
-  {0, 0},
-  {0, 0},
-  {0, 0},
-  {0, 0},
-  {0, 0},
-  {0, 0},
-  {0, pg_sjis_mblen}
-};
-
-/*
- *########################################################################
- *
- * Public functions
- *
- *########################################################################
- */
-
-/* convert a multi-byte string to a wchar */
-void pg_mb2wchar(const unsigned char *from, pg_wchar *to)
-{
-  (*pg_wchar_table[MULTIBYTE].mb2wchar_with_len)(from,to,strlen(from));
-}
-
-/* convert a multi-byte string to a wchar with a limited length */
-void pg_mb2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
-{
-  (*pg_wchar_table[MULTIBYTE].mb2wchar_with_len)(from,to,len);
-}
-
-/* returns the byte length of a multi-byte word */
-int pg_mblen(const unsigned char *mbstr)
-{
-  return((*pg_wchar_table[MULTIBYTE].mblen)(mbstr));
-}
-
-/* returns the byte length of a multi-byte word for an encoding */
-int pg_encoding_mblen(int encoding, const unsigned char *mbstr)
-{
-  return((*pg_wchar_table[encoding].mblen)(mbstr));
-}
-
-/* returns the byte length of a word for mule internal code */
-int pg_mic_mblen(const unsigned char *mbstr)
-{
-  return(pg_mule_mblen(mbstr));
-}
-
-/* returns the length (counted as a wchar) of a multi-byte string */
-int pg_mbstrlen(const unsigned char *mbstr)
-{
-  int len = 0;
-  while (*mbstr) {
-    mbstr += pg_mblen(mbstr);
-    len++;
-  }
-  return(len);
-}
-
-/* returns the length (counted as a wchar) of a multi-byte string 
-   (not necessarily  NULL terminated) */
-int pg_mbstrlen_with_len(const unsigned char *mbstr, int limit)
-{
-  int len = 0;
-  int l;
-  while (*mbstr && limit > 0) {
-    l = pg_mblen(mbstr);
-    limit -= l;
-    mbstr += l;
-    len++;
-  }
-  return(len);
-}
diff --git a/src/backend/regex/wstrcmp.c b/src/backend/regex/wstrcmp.c
deleted file mode 100644
index b562f103150..00000000000
--- a/src/backend/regex/wstrcmp.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*-
- * Copyright (c) 1990, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * This code is derived from software contributed to Berkeley by
- * Chris Torek.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <regex/pg_wchar.h>
-
-int
-pg_char_and_wchar_strcmp(s1, s2)
-	register const char *s1;
-	register const pg_wchar *s2;
-{
-	while ((pg_wchar)*s1 == *s2++)
-		if (*s1++ == 0)
-			return (0);
-	return (*(const unsigned char *)s1 - *(const pg_wchar *)(s2 - 1));
-}
diff --git a/src/backend/regex/wstrncmp.c b/src/backend/regex/wstrncmp.c
deleted file mode 100644
index e7ce52ed7b2..00000000000
--- a/src/backend/regex/wstrncmp.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright (c) 1989, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * This code is derived from FreeBSD 2.2.1-RELEASE software.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <regex/pg_wchar.h>
-
-int
-pg_wchar_strncmp(s1, s2, n)
-	register const pg_wchar *s1, *s2;
-	register size_t n;
-{
-
-	if (n == 0)
-		return (0);
-	do {
-		if (*s1 != *s2++)
-			return (*(const pg_wchar *)s1 -
-				*(const pg_wchar *)(s2 - 1));
-		if (*s1++ == 0)
-			break;
-	} while (--n != 0);
-	return (0);
-}
-
-int
-pg_char_and_wchar_strncmp(s1, s2, n)
-	register const char *s1;
-	register const pg_wchar *s2;
-	register size_t n;
-{
-
-	if (n == 0)
-		return (0);
-	do {
-		if ((pg_wchar )*s1 != *s2++)
-			return (*(const pg_wchar *)s1 -
-				*(const pg_wchar *)(s2 - 1));
-		if (*s1++ == 0)
-			break;
-	} while (--n != 0);
-	return (0);
-}
-
-size_t
-pg_wchar_strlen(str)
-	const pg_wchar *str;
-{
-	register const pg_wchar *s;
-
-	for (s = str; *s; ++s);
-	return(s - str);
-}