Ensure that btree sort ordering functions and boolean comparison operators

give consistent results for all datatypes.  Types float4, float8, and
numeric were broken for NaN values; abstime, timestamp, and interval
were broken for INVALID values; timetz was just plain broken (some
possible pairs of values were neither < nor = nor >).  Also clean up
text, bpchar, varchar, and bit/varbit to eliminate duplicate code and
thereby reduce the probability of similar inconsistencies arising in
the future.

1 files changed, 40 insertions, 116 deletions

diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c
index d4e93cf8756..99df5331bf6 100644
--- a/src/backend/utils/adt/numeric.c
+++ b/src/backend/utils/adt/numeric.c
@@ -5,7 +5,7 @@
  *
  *	1998 Jan Wieck
  *
- * $Header: /cvsroot/pgsql/src/backend/utils/adt/numeric.c,v 1.40 2001/04/14 02:10:57 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/utils/adt/numeric.c,v 1.41 2001/05/03 19:00:36 tgl Exp $
  *
  * ----------
  */
@@ -153,6 +153,7 @@ static Numeric make_result(NumericVar *var);
 
 static void apply_typmod(NumericVar *var, int32 typmod);
 
+static int	cmp_numerics(Numeric num1, Numeric num2);
 static int	cmp_var(NumericVar *var1, NumericVar *var2);
 static void add_var(NumericVar *var1, NumericVar *var2, NumericVar *result);
 static void sub_var(NumericVar *var1, NumericVar *var2, NumericVar *result);
@@ -664,24 +665,7 @@ numeric_cmp(PG_FUNCTION_ARGS)
 	Numeric		num2 = PG_GETARG_NUMERIC(1);
 	int			result;
 
-	if (NUMERIC_IS_NAN(num1) || NUMERIC_IS_NAN(num2))
-		result = 0;
-	else
-	{
-		NumericVar	arg1;
-		NumericVar	arg2;
-
-		init_var(&arg1);
-		init_var(&arg2);
-
-		set_var_from_num(num1, &arg1);
-		set_var_from_num(num2, &arg2);
-
-		result = cmp_var(&arg1, &arg2);
-
-		free_var(&arg1);
-		free_var(&arg2);
-	}
+	result = cmp_numerics(num1, num2);
 
 	PG_FREE_IF_COPY(num1, 0);
 	PG_FREE_IF_COPY(num2, 1);
@@ -697,24 +681,7 @@ numeric_eq(PG_FUNCTION_ARGS)
 	Numeric		num2 = PG_GETARG_NUMERIC(1);
 	bool		result;
 
-	if (NUMERIC_IS_NAN(num1) || NUMERIC_IS_NAN(num2))
-		result = false;
-	else
-	{
-		NumericVar	arg1;
-		NumericVar	arg2;
-
-		init_var(&arg1);
-		init_var(&arg2);
-
-		set_var_from_num(num1, &arg1);
-		set_var_from_num(num2, &arg2);
-
-		result = cmp_var(&arg1, &arg2) == 0;
-
-		free_var(&arg1);
-		free_var(&arg2);
-	}
+	result = cmp_numerics(num1, num2) == 0;
 
 	PG_FREE_IF_COPY(num1, 0);
 	PG_FREE_IF_COPY(num2, 1);
@@ -729,24 +696,7 @@ numeric_ne(PG_FUNCTION_ARGS)
 	Numeric		num2 = PG_GETARG_NUMERIC(1);
 	bool		result;
 
-	if (NUMERIC_IS_NAN(num1) || NUMERIC_IS_NAN(num2))
-		result = false;
-	else
-	{
-		NumericVar	arg1;
-		NumericVar	arg2;
-
-		init_var(&arg1);
-		init_var(&arg2);
-
-		set_var_from_num(num1, &arg1);
-		set_var_from_num(num2, &arg2);
-
-		result = cmp_var(&arg1, &arg2) != 0;
-
-		free_var(&arg1);
-		free_var(&arg2);
-	}
+	result = cmp_numerics(num1, num2) != 0;
 
 	PG_FREE_IF_COPY(num1, 0);
 	PG_FREE_IF_COPY(num2, 1);
@@ -761,24 +711,7 @@ numeric_gt(PG_FUNCTION_ARGS)
 	Numeric		num2 = PG_GETARG_NUMERIC(1);
 	bool		result;
 
-	if (NUMERIC_IS_NAN(num1) || NUMERIC_IS_NAN(num2))
-		result = false;
-	else
-	{
-		NumericVar	arg1;
-		NumericVar	arg2;
-
-		init_var(&arg1);
-		init_var(&arg2);
-
-		set_var_from_num(num1, &arg1);
-		set_var_from_num(num2, &arg2);
-
-		result = cmp_var(&arg1, &arg2) > 0;
-
-		free_var(&arg1);
-		free_var(&arg2);
-	}
+	result = cmp_numerics(num1, num2) > 0;
 
 	PG_FREE_IF_COPY(num1, 0);
 	PG_FREE_IF_COPY(num2, 1);
@@ -793,24 +726,7 @@ numeric_ge(PG_FUNCTION_ARGS)
 	Numeric		num2 = PG_GETARG_NUMERIC(1);
 	bool		result;
 
-	if (NUMERIC_IS_NAN(num1) || NUMERIC_IS_NAN(num2))
-		result = false;
-	else
-	{
-		NumericVar	arg1;
-		NumericVar	arg2;
-
-		init_var(&arg1);
-		init_var(&arg2);
-
-		set_var_from_num(num1, &arg1);
-		set_var_from_num(num2, &arg2);
-
-		result = cmp_var(&arg1, &arg2) >= 0;
-
-		free_var(&arg1);
-		free_var(&arg2);
-	}
+	result = cmp_numerics(num1, num2) >= 0;
 
 	PG_FREE_IF_COPY(num1, 0);
 	PG_FREE_IF_COPY(num2, 1);
@@ -825,24 +741,7 @@ numeric_lt(PG_FUNCTION_ARGS)
 	Numeric		num2 = PG_GETARG_NUMERIC(1);
 	bool		result;
 
-	if (NUMERIC_IS_NAN(num1) || NUMERIC_IS_NAN(num2))
-		result = false;
-	else
-	{
-		NumericVar	arg1;
-		NumericVar	arg2;
-
-		init_var(&arg1);
-		init_var(&arg2);
-
-		set_var_from_num(num1, &arg1);
-		set_var_from_num(num2, &arg2);
-
-		result = cmp_var(&arg1, &arg2) < 0;
-
-		free_var(&arg1);
-		free_var(&arg2);
-	}
+	result = cmp_numerics(num1, num2) < 0;
 
 	PG_FREE_IF_COPY(num1, 0);
 	PG_FREE_IF_COPY(num2, 1);
@@ -857,8 +756,35 @@ numeric_le(PG_FUNCTION_ARGS)
 	Numeric		num2 = PG_GETARG_NUMERIC(1);
 	bool		result;
 
-	if (NUMERIC_IS_NAN(num1) || NUMERIC_IS_NAN(num2))
-		result = false;
+	result = cmp_numerics(num1, num2) <= 0;
+
+	PG_FREE_IF_COPY(num1, 0);
+	PG_FREE_IF_COPY(num2, 1);
+
+	PG_RETURN_BOOL(result);
+}
+
+static int
+cmp_numerics(Numeric num1, Numeric num2)
+{
+	int			result;
+
+	/*
+	 * We consider all NANs to be equal and larger than any non-NAN.
+	 * This is somewhat arbitrary; the important thing is to have a
+	 * consistent sort order.
+	 */
+	if (NUMERIC_IS_NAN(num1))
+	{
+		if (NUMERIC_IS_NAN(num2))
+			result = 0;			/* NAN = NAN */
+		else
+			result = 1;			/* NAN > non-NAN */
+	}
+	else if (NUMERIC_IS_NAN(num2))
+	{
+		result = -1;			/* non-NAN < NAN */
+	}
 	else
 	{
 		NumericVar	arg1;
@@ -870,16 +796,13 @@ numeric_le(PG_FUNCTION_ARGS)
 		set_var_from_num(num1, &arg1);
 		set_var_from_num(num2, &arg2);
 
-		result = cmp_var(&arg1, &arg2) <= 0;
+		result = cmp_var(&arg1, &arg2);
 
 		free_var(&arg1);
 		free_var(&arg2);
 	}
 
-	PG_FREE_IF_COPY(num1, 0);
-	PG_FREE_IF_COPY(num2, 1);
-
-	PG_RETURN_BOOL(result);
+	return result;
 }
 
 
@@ -1663,6 +1586,7 @@ numeric_int2(PG_FUNCTION_ARGS)
 	char	   *str;
 	Datum		result;
 
+	/* XXX would it be better to return NULL? */
 	if (NUMERIC_IS_NAN(num))
 		elog(ERROR, "Cannot convert NaN to int2");