diff options
| author | Tom Lane <tgl@sss.pgh.pa.us> | 2001-12-11 02:02:12 +0000 |
|---|---|---|
| committer | Tom Lane <tgl@sss.pgh.pa.us> | 2001-12-11 02:02:12 +0000 |
| commit | 07009651ce4b2cc866171aed32bb67c8c65137ce (patch) | |
| tree | 8ac63ba286fe8e9e5f6cfbf8d97b8e480e2f191c /src/backend/utils/adt/float.c | |
| parent | 63cc56de54049e9e2c16dde182fb93c09298af3b (diff) | |
| download | postgresql-07009651ce4b2cc866171aed32bb67c8c65137ce.tar.gz postgresql-07009651ce4b2cc866171aed32bb67c8c65137ce.zip | |
Repair roundoff-error problem for stddev/variance results near zero,
per complaint from Kemin Zhou.
Fix lack of precision in numeric stddev/variance.
Diffstat (limited to 'src/backend/utils/adt/float.c')
| -rw-r--r-- | src/backend/utils/adt/float.c | 24 |
1 files changed, 19 insertions, 5 deletions
diff --git a/src/backend/utils/adt/float.c b/src/backend/utils/adt/float.c index 7d2018d7470..0f30fc63efb 100644 --- a/src/backend/utils/adt/float.c +++ b/src/backend/utils/adt/float.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $Header: /cvsroot/pgsql/src/backend/utils/adt/float.c,v 1.77 2001/11/05 17:46:29 momjian Exp $ + * $Header: /cvsroot/pgsql/src/backend/utils/adt/float.c,v 1.78 2001/12/11 02:02:12 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -1580,7 +1580,8 @@ float8_variance(PG_FUNCTION_ARGS) float8 *transvalues; float8 N, sumX, - sumX2; + sumX2, + numerator; transvalues = check_float8_array(transarray, "float8_variance"); N = transvalues[0]; @@ -1594,7 +1595,13 @@ float8_variance(PG_FUNCTION_ARGS) if (N <= 1.0) PG_RETURN_FLOAT8(0.0); - PG_RETURN_FLOAT8((N * sumX2 - sumX * sumX) / (N * (N - 1.0))); + numerator = N * sumX2 - sumX * sumX; + + /* Watch out for roundoff error producing a negative numerator */ + if (numerator <= 0.0) + PG_RETURN_FLOAT8(0.0); + + PG_RETURN_FLOAT8(numerator / (N * (N - 1.0))); } Datum @@ -1604,7 +1611,8 @@ float8_stddev(PG_FUNCTION_ARGS) float8 *transvalues; float8 N, sumX, - sumX2; + sumX2, + numerator; transvalues = check_float8_array(transarray, "float8_stddev"); N = transvalues[0]; @@ -1618,7 +1626,13 @@ float8_stddev(PG_FUNCTION_ARGS) if (N <= 1.0) PG_RETURN_FLOAT8(0.0); - PG_RETURN_FLOAT8(sqrt((N * sumX2 - sumX * sumX) / (N * (N - 1.0)))); + numerator = N * sumX2 - sumX * sumX; + + /* Watch out for roundoff error producing a negative numerator */ + if (numerator <= 0.0) + PG_RETURN_FLOAT8(0.0); + + PG_RETURN_FLOAT8(sqrt(numerator / (N * (N - 1.0)))); } |