diff options
author | Tom Lane <tgl@sss.pgh.pa.us> | 2020-03-16 21:05:28 -0400 |
---|---|---|
committer | Tom Lane <tgl@sss.pgh.pa.us> | 2020-03-16 21:05:52 -0400 |
commit | b4570d33aa045df330bb325ba8a2cbf02266a555 (patch) | |
tree | 4e7ebfee102862d095bfa9eb0dede58a4cca471f /src | |
parent | 113758155c11cf993ca0ecee8856e300a2525a30 (diff) | |
download | postgresql-b4570d33aa045df330bb325ba8a2cbf02266a555.tar.gz postgresql-b4570d33aa045df330bb325ba8a2cbf02266a555.zip |
Avoid holding a directory FD open across assorted SRF calls.
This extends the fixes made in commit 085b6b667 to other SRFs with the
same bug, namely pg_logdir_ls(), pgrowlocks(), pg_timezone_names(),
pg_ls_dir(), and pg_tablespace_databases().
Also adjust various comments and documentation to warn against
expecting to clean up resources during a ValuePerCall SRF's final
call.
Back-patch to all supported branches, since these functions were
all born broken.
Justin Pryzby, with cosmetic tweaks by me
Discussion: https://postgr.es/m/20200308173103.GC1357@telsasoft.com
Diffstat (limited to 'src')
-rw-r--r-- | src/backend/utils/adt/datetime.c | 101 | ||||
-rw-r--r-- | src/backend/utils/adt/genfile.c | 112 | ||||
-rw-r--r-- | src/backend/utils/adt/misc.c | 117 | ||||
-rw-r--r-- | src/backend/utils/fmgr/README | 16 | ||||
-rw-r--r-- | src/include/funcapi.h | 13 | ||||
-rw-r--r-- | src/test/regress/expected/misc_functions.out | 21 | ||||
-rw-r--r-- | src/test/regress/sql/misc_functions.sql | 9 |
7 files changed, 213 insertions, 176 deletions
diff --git a/src/backend/utils/adt/datetime.c b/src/backend/utils/adt/datetime.c index 4f109111d19..9c808942819 100644 --- a/src/backend/utils/adt/datetime.c +++ b/src/backend/utils/adt/datetime.c @@ -4755,12 +4755,12 @@ pg_timezone_abbrevs(PG_FUNCTION_ARGS) Datum pg_timezone_names(PG_FUNCTION_ARGS) { - MemoryContext oldcontext; - FuncCallContext *funcctx; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + bool randomAccess; + TupleDesc tupdesc; + Tuplestorestate *tupstore; pg_tzenum *tzenum; pg_tz *tz; - Datum result; - HeapTuple tuple; Datum values[4]; bool nulls[4]; int tzoff; @@ -4769,59 +4769,41 @@ pg_timezone_names(PG_FUNCTION_ARGS) const char *tzn; Interval *resInterval; struct pg_tm itm; + MemoryContext oldcontext; - /* stuff done only on the first call of the function */ - if (SRF_IS_FIRSTCALL()) - { - TupleDesc tupdesc; + /* check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("materialize mode required, but it is not allowed in this context"))); - /* create a function context for cross-call persistence */ - funcctx = SRF_FIRSTCALL_INIT(); + /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ + oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - /* - * switch to memory context appropriate for multiple function calls - */ - oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); - /* initialize timezone scanning code */ - tzenum = pg_tzenumerate_start(); - funcctx->user_fctx = (void *) tzenum; + randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; + tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; - /* - * build tupdesc for result tuples. This must match this function's - * pg_proc entry! - */ - tupdesc = CreateTemplateTupleDesc(4); - TupleDescInitEntry(tupdesc, (AttrNumber) 1, "name", - TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 2, "abbrev", - TEXTOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 3, "utc_offset", - INTERVALOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 4, "is_dst", - BOOLOID, -1, 0); + MemoryContextSwitchTo(oldcontext); - funcctx->tuple_desc = BlessTupleDesc(tupdesc); - MemoryContextSwitchTo(oldcontext); - } - - /* stuff done on every call of the function */ - funcctx = SRF_PERCALL_SETUP(); - tzenum = (pg_tzenum *) funcctx->user_fctx; + /* initialize timezone scanning code */ + tzenum = pg_tzenumerate_start(); /* search for another zone to display */ for (;;) { - oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); tz = pg_tzenumerate_next(tzenum); - MemoryContextSwitchTo(oldcontext); - if (!tz) - { - pg_tzenumerate_end(tzenum); - funcctx->user_fctx = NULL; - SRF_RETURN_DONE(funcctx); - } + break; /* Convert now() to local time in this zone */ if (timestamp2tm(GetCurrentTransactionStartTimestamp(), @@ -4840,25 +4822,22 @@ pg_timezone_names(PG_FUNCTION_ARGS) if (tzn && strlen(tzn) > 31) continue; - /* Found a displayable zone */ - break; - } + MemSet(nulls, 0, sizeof(nulls)); - MemSet(nulls, 0, sizeof(nulls)); + values[0] = CStringGetTextDatum(pg_get_timezone_name(tz)); + values[1] = CStringGetTextDatum(tzn ? tzn : ""); - values[0] = CStringGetTextDatum(pg_get_timezone_name(tz)); - values[1] = CStringGetTextDatum(tzn ? tzn : ""); + MemSet(&itm, 0, sizeof(struct pg_tm)); + itm.tm_sec = -tzoff; + resInterval = (Interval *) palloc(sizeof(Interval)); + tm2interval(&itm, 0, resInterval); + values[2] = IntervalPGetDatum(resInterval); - MemSet(&itm, 0, sizeof(struct pg_tm)); - itm.tm_sec = -tzoff; - resInterval = (Interval *) palloc(sizeof(Interval)); - tm2interval(&itm, 0, resInterval); - values[2] = IntervalPGetDatum(resInterval); + values[3] = BoolGetDatum(tm.tm_isdst > 0); - values[3] = BoolGetDatum(tm.tm_isdst > 0); - - tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); - result = HeapTupleGetDatum(tuple); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } - SRF_RETURN_NEXT(funcctx, result); + pg_tzenumerate_end(tzenum); + return (Datum) 0; } diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index bcf9bd1b970..01185f218b7 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -36,13 +36,6 @@ #include "utils/syscache.h" #include "utils/timestamp.h" -typedef struct -{ - char *location; - DIR *dirdesc; - bool include_dot_dirs; -} directory_fctx; - /* * Convert a "text" filename argument to C string, and check it's allowable. @@ -447,67 +440,79 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS) Datum pg_ls_dir(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + char *location; + bool missing_ok = false; + bool include_dot_dirs = false; + bool randomAccess; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + DIR *dirdesc; struct dirent *de; - directory_fctx *fctx; MemoryContext oldcontext; - if (SRF_IS_FIRSTCALL()) + location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + + /* check the optional arguments */ + if (PG_NARGS() == 3) { - bool missing_ok = false; - bool include_dot_dirs = false; + if (!PG_ARGISNULL(1)) + missing_ok = PG_GETARG_BOOL(1); + if (!PG_ARGISNULL(2)) + include_dot_dirs = PG_GETARG_BOOL(2); + } - /* check the optional arguments */ - if (PG_NARGS() == 3) - { - if (!PG_ARGISNULL(1)) - missing_ok = PG_GETARG_BOOL(1); - if (!PG_ARGISNULL(2)) - include_dot_dirs = PG_GETARG_BOOL(2); - } + /* check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("materialize mode required, but it is not allowed in this context"))); + + /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ + oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - funcctx = SRF_FIRSTCALL_INIT(); - oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0); - fctx = palloc(sizeof(directory_fctx)); - fctx->location = convert_and_check_filename(PG_GETARG_TEXT_PP(0)); + randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; + tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; - fctx->include_dot_dirs = include_dot_dirs; - fctx->dirdesc = AllocateDir(fctx->location); + MemoryContextSwitchTo(oldcontext); - if (!fctx->dirdesc) - { - if (missing_ok && errno == ENOENT) - { - MemoryContextSwitchTo(oldcontext); - SRF_RETURN_DONE(funcctx); - } - else - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open directory \"%s\": %m", - fctx->location))); - } - funcctx->user_fctx = fctx; - MemoryContextSwitchTo(oldcontext); + dirdesc = AllocateDir(location); + if (!dirdesc) + { + /* Return empty tuplestore if appropriate */ + if (missing_ok && errno == ENOENT) + return (Datum) 0; + /* Otherwise, we can let ReadDir() throw the error */ } - funcctx = SRF_PERCALL_SETUP(); - fctx = (directory_fctx *) funcctx->user_fctx; - - while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL) + while ((de = ReadDir(dirdesc, location)) != NULL) { - if (!fctx->include_dot_dirs && + Datum values[1]; + bool nulls[1]; + + if (!include_dot_dirs && (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)) continue; - SRF_RETURN_NEXT(funcctx, CStringGetTextDatum(de->d_name)); - } + values[0] = CStringGetTextDatum(de->d_name); + nulls[0] = false; - FreeDir(fctx->dirdesc); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } - SRF_RETURN_DONE(funcctx); + FreeDir(dirdesc); + return (Datum) 0; } /* @@ -548,8 +553,7 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) if (!(rsinfo->allowedModes & SFRM_Materialize)) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("materialize mode required, but it is not " - "allowed in this context"))); + errmsg("materialize mode required, but it is not allowed in this context"))); /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); @@ -575,10 +579,7 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) { /* Return empty tuplestore if appropriate */ if (missing_ok && errno == ENOENT) - { - tuplestore_donestoring(tupstore); return (Datum) 0; - } /* Otherwise, we can let ReadDir() throw the error */ } @@ -613,7 +614,6 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok) } FreeDir(dirdesc); - tuplestore_donestoring(tupstore); return (Datum) 0; } diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c index 323e36b81c7..ee340fb0f02 100644 --- a/src/backend/utils/adt/misc.c +++ b/src/backend/utils/adt/misc.c @@ -194,72 +194,82 @@ current_query(PG_FUNCTION_ARGS) /* Function to find out which databases make use of a tablespace */ -typedef struct -{ - char *location; - DIR *dirdesc; -} ts_db_fctx; - Datum pg_tablespace_databases(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; + Oid tablespaceOid = PG_GETARG_OID(0); + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + bool randomAccess; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + char *location; + DIR *dirdesc; struct dirent *de; - ts_db_fctx *fctx; + MemoryContext oldcontext; - if (SRF_IS_FIRSTCALL()) - { - MemoryContext oldcontext; - Oid tablespaceOid = PG_GETARG_OID(0); + /* check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("materialize mode required, but it is not allowed in this context"))); - funcctx = SRF_FIRSTCALL_INIT(); - oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + /* The tupdesc and tuplestore must be created in ecxt_per_query_memory */ + oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); - fctx = palloc(sizeof(ts_db_fctx)); + tupdesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_tablespace_databases", + OIDOID, -1, 0); - if (tablespaceOid == GLOBALTABLESPACE_OID) - { - fctx->dirdesc = NULL; - ereport(WARNING, - (errmsg("global tablespace never has databases"))); - } - else - { - if (tablespaceOid == DEFAULTTABLESPACE_OID) - fctx->location = psprintf("base"); - else - fctx->location = psprintf("pg_tblspc/%u/%s", tablespaceOid, - TABLESPACE_VERSION_DIRECTORY); + randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; + tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); - fctx->dirdesc = AllocateDir(fctx->location); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; - if (!fctx->dirdesc) - { - /* the only expected error is ENOENT */ - if (errno != ENOENT) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open directory \"%s\": %m", - fctx->location))); - ereport(WARNING, - (errmsg("%u is not a tablespace OID", tablespaceOid))); - } - } - funcctx->user_fctx = fctx; - MemoryContextSwitchTo(oldcontext); + MemoryContextSwitchTo(oldcontext); + + if (tablespaceOid == GLOBALTABLESPACE_OID) + { + ereport(WARNING, + (errmsg("global tablespace never has databases"))); + /* return empty tuplestore */ + return (Datum) 0; } - funcctx = SRF_PERCALL_SETUP(); - fctx = (ts_db_fctx *) funcctx->user_fctx; + if (tablespaceOid == DEFAULTTABLESPACE_OID) + location = psprintf("base"); + else + location = psprintf("pg_tblspc/%u/%s", tablespaceOid, + TABLESPACE_VERSION_DIRECTORY); - if (!fctx->dirdesc) /* not a tablespace */ - SRF_RETURN_DONE(funcctx); + dirdesc = AllocateDir(location); - while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL) + if (!dirdesc) + { + /* the only expected error is ENOENT */ + if (errno != ENOENT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open directory \"%s\": %m", + location))); + ereport(WARNING, + (errmsg("%u is not a tablespace OID", tablespaceOid))); + /* return empty tuplestore */ + return (Datum) 0; + } + + while ((de = ReadDir(dirdesc, location)) != NULL) { Oid datOid = atooid(de->d_name); char *subdir; bool isempty; + Datum values[1]; + bool nulls[1]; /* this test skips . and .., but is awfully weak */ if (!datOid) @@ -267,18 +277,21 @@ pg_tablespace_databases(PG_FUNCTION_ARGS) /* if database subdir is empty, don't report tablespace as used */ - subdir = psprintf("%s/%s", fctx->location, de->d_name); + subdir = psprintf("%s/%s", location, de->d_name); isempty = directory_is_empty(subdir); pfree(subdir); if (isempty) continue; /* indeed, nothing in it */ - SRF_RETURN_NEXT(funcctx, ObjectIdGetDatum(datOid)); + values[0] = ObjectIdGetDatum(datOid); + nulls[0] = false; + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); } - FreeDir(fctx->dirdesc); - SRF_RETURN_DONE(funcctx); + FreeDir(dirdesc); + return (Datum) 0; } diff --git a/src/backend/utils/fmgr/README b/src/backend/utils/fmgr/README index a4d6a07bddc..1e4c4b94a95 100644 --- a/src/backend/utils/fmgr/README +++ b/src/backend/utils/fmgr/README @@ -239,8 +239,6 @@ tuple toaster will decide whether toasting is needed. Functions Accepting or Returning Sets ------------------------------------- -[ this section revised 29-Aug-2002 for 7.3 ] - If a function is marked in pg_proc as returning a set, then it is called with fcinfo->resultinfo pointing to a node of type ReturnSetInfo. A function that desires to return a set should raise an error "called in @@ -277,10 +275,16 @@ been returned, the next call should set isDone to ExprEndResult and return a null result. (Note it is possible to return an empty set by doing this on the first call.) -The ReturnSetInfo node also contains a link to the ExprContext within which -the function is being evaluated. This is useful for value-per-call functions -that need to close down internal state when they are not run to completion: -they can register a shutdown callback function in the ExprContext. +Value-per-call functions MUST NOT assume that they will be run to completion; +the executor might simply stop calling them, for example because of a LIMIT. +Therefore, it's unsafe to attempt to perform any resource cleanup in the +final call. It's usually not necessary to clean up memory, anyway. If it's +necessary to clean up other types of resources, such as file descriptors, +one can register a shutdown callback function in the ExprContext pointed to +by the ReturnSetInfo node. (But note that file descriptors are a limited +resource, so it's generally unwise to hold those open across calls; SRFs +that need file access are better written to do it in a single call using +Materialize mode.) Materialize mode works like this: the function creates a Tuplestore holding the (possibly empty) result set, and returns it. There are no multiple calls. diff --git a/src/include/funcapi.h b/src/include/funcapi.h index f9b75ae3905..b047acdc1a8 100644 --- a/src/include/funcapi.h +++ b/src/include/funcapi.h @@ -234,7 +234,7 @@ extern Datum HeapTupleHeaderGetDatum(HeapTupleHeader tuple); /*---------- * Support for Set Returning Functions (SRFs) * - * The basic API for SRFs looks something like: + * The basic API for SRFs using ValuePerCall mode looks something like this: * * Datum * my_Set_Returning_Function(PG_FUNCTION_ARGS) @@ -271,6 +271,17 @@ extern Datum HeapTupleHeaderGetDatum(HeapTupleHeader tuple); * SRF_RETURN_DONE(funcctx); * } * + * NOTE: there is no guarantee that a SRF using ValuePerCall mode will be + * run to completion; for example, a query with LIMIT might stop short of + * fetching all the rows. Therefore, do not expect that you can do resource + * cleanup just before SRF_RETURN_DONE(). You need not worry about releasing + * memory allocated in multi_call_memory_ctx, but holding file descriptors or + * other non-memory resources open across calls is a bug. SRFs that need + * such resources should not use these macros, but instead populate a + * tuplestore during a single call, and return that using SFRM_Materialize + * mode (see fmgr/README). Alternatively, set up a callback to release + * resources at query shutdown, using RegisterExprContextCallback(). + * *---------- */ diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index e217b678d78..d3acb98d04e 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -180,6 +180,27 @@ select count(*) >= 0 as ok from pg_ls_archive_statusdir(); t (1 row) +select * from (select pg_ls_dir('.') a) a where a = 'base' limit 1; + a +------ + base +(1 row) + +select * from (select (pg_timezone_names()).name) ptn where name='UTC' limit 1; + name +------ + UTC +(1 row) + +select count(*) > 0 from + (select pg_tablespace_databases(oid) as pts from pg_tablespace + where spcname = 'pg_default') pts + join pg_database db on pts.pts = db.oid; + ?column? +---------- + t +(1 row) + -- -- Test adding a support function to a subject function -- diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 1e11eb35547..094e8f8296f 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -51,6 +51,15 @@ from (select pg_ls_waldir() w) ss where length((w).name) = 24 limit 1; select count(*) >= 0 as ok from pg_ls_archive_statusdir(); +select * from (select pg_ls_dir('.') a) a where a = 'base' limit 1; + +select * from (select (pg_timezone_names()).name) ptn where name='UTC' limit 1; + +select count(*) > 0 from + (select pg_tablespace_databases(oid) as pts from pg_tablespace + where spcname = 'pg_default') pts + join pg_database db on pts.pts = db.oid; + -- -- Test adding a support function to a subject function -- |