Tweak sorting so that nulls appear at the front of a descending sort

(vs. at the end of a normal sort).  This ensures that explicit sorts
yield the same ordering as a btree index scan.  To be really sure that
that equivalence holds, we use the btree entries in pg_amop to decide
whether we are looking at a '<' or '>' operator.  For a sort operator
that has no btree association, we put the nulls at the front if the
operator is named '>' ... pretty grotty, but it does the right thing in
simple ASC and DESC cases, and at least there's no possibility of getting
a different answer depending on the plan type chosen.

1 files changed, 6 insertions, 21 deletions

diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 24cc7a8b254..c5f2799022a 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/commands/analyze.c,v 1.17 2001/05/07 00:43:17 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/commands/analyze.c,v 1.18 2001/06/02 19:01:53 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1439,27 +1439,12 @@ compare_scalars(const void *a, const void *b)
 	int			ta = ((ScalarItem *) a)->tupno;
 	Datum		db = ((ScalarItem *) b)->value;
 	int			tb = ((ScalarItem *) b)->tupno;
+	int32		compare;
 
-	if (datumCmpFnKind == SORTFUNC_LT)
-	{
-		if (DatumGetBool(FunctionCall2(datumCmpFn, da, db)))
-			return -1;			/* a < b */
-		if (DatumGetBool(FunctionCall2(datumCmpFn, db, da)))
-			return 1;			/* a > b */
-	}
-	else
-	{
-		/* sort function is CMP or REVCMP */
-		int32	compare;
-
-		compare = DatumGetInt32(FunctionCall2(datumCmpFn, da, db));
-		if (compare != 0)
-		{
-			if (datumCmpFnKind == SORTFUNC_REVCMP)
-				compare = -compare;
-			return compare;
-		}
-	}
+	compare = ApplySortFunction(datumCmpFn, datumCmpFnKind,
+								da, false, db, false);
+	if (compare != 0)
+		return compare;
 
 	/*
 	 * The two datums are equal, so update datumCmpTupnoLink[].