diff options
| author | Tom Lane <tgl@sss.pgh.pa.us> | 2007-07-09 16:14:06 +0000 |
|---|---|---|
| committer | Tom Lane <tgl@sss.pgh.pa.us> | 2007-07-09 16:14:06 +0000 |
| commit | 3df98a9e2fcada34de3ec8325f83b82426014c6d (patch) | |
| tree | aa6a65a609aa462e43f25b2305d005b39058304c | |
| parent | 809b38ce27742b1e47acad44c3742ea5af6add8d (diff) | |
| download | postgresql-3df98a9e2fcada34de3ec8325f83b82426014c6d.tar.gz postgresql-3df98a9e2fcada34de3ec8325f83b82426014c6d.zip | |
Fix stddev_pop(numeric) and var_pop(numeric), which were incorrectly producing
the same outputs as stddev_samp() and var_samp() respectively.
| -rw-r--r-- | src/backend/utils/adt/numeric.c | 7 | ||||
| -rw-r--r-- | src/test/regress/expected/aggregates.out | 4 |
2 files changed, 7 insertions, 4 deletions
diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index 71a2e491868..65530f9dd8e 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -14,7 +14,7 @@ * Copyright (c) 1998-2006, PostgreSQL Global Development Group * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/numeric.c,v 1.96.2.1 2007/06/09 15:52:38 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/numeric.c,v 1.96.2.2 2007/07/09 16:14:05 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -2274,7 +2274,10 @@ numeric_stddev_internal(ArrayType *transarray, } else { - mul_var(&vN, &vNminus1, &vNminus1, 0); /* N * (N - 1) */ + if (sample) + mul_var(&vN, &vNminus1, &vNminus1, 0); /* N * (N - 1) */ + else + mul_var(&vN, &vN, &vNminus1, 0); /* N * N */ rscale = select_div_scale(&vsumX2, &vNminus1); div_var(&vsumX2, &vNminus1, &vsumX, rscale, true); /* variance */ if (!variance) diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out index 34b720f0f43..74635479e48 100644 --- a/src/test/regress/expected/aggregates.out +++ b/src/test/regress/expected/aggregates.out @@ -102,7 +102,7 @@ SELECT var_samp(b) FROM aggtest; SELECT stddev_pop(b::numeric) FROM aggtest; stddev_pop ------------------ - 151.389361431288 + 131.107032862199 (1 row) SELECT stddev_samp(b::numeric) FROM aggtest; @@ -114,7 +114,7 @@ SELECT stddev_samp(b::numeric) FROM aggtest; SELECT var_pop(b::numeric) FROM aggtest; var_pop -------------------- - 22918.738754573025 + 17189.054065929769 (1 row) SELECT var_samp(b::numeric) FROM aggtest; |