diff options
| -rw-r--r-- | contrib/file_fdw/file_fdw.c | 161 | ||||
| -rw-r--r-- | doc/src/sgml/fdwhandler.sgml | 230 | ||||
| -rw-r--r-- | src/backend/nodes/copyfuncs.c | 3 | ||||
| -rw-r--r-- | src/backend/nodes/outfuncs.c | 4 | ||||
| -rw-r--r-- | src/backend/optimizer/path/allpaths.c | 13 | ||||
| -rw-r--r-- | src/backend/optimizer/path/costsize.c | 2 | ||||
| -rw-r--r-- | src/backend/optimizer/plan/createplan.c | 67 | ||||
| -rw-r--r-- | src/backend/optimizer/plan/setrefs.c | 2 | ||||
| -rw-r--r-- | src/backend/optimizer/plan/subselect.c | 2 | ||||
| -rw-r--r-- | src/backend/optimizer/util/pathnode.c | 2 | ||||
| -rw-r--r-- | src/backend/optimizer/util/relnode.c | 4 | ||||
| -rw-r--r-- | src/include/foreign/fdwapi.h | 21 | ||||
| -rw-r--r-- | src/include/nodes/plannodes.h | 11 | ||||
| -rw-r--r-- | src/include/nodes/relation.h | 24 | ||||
| -rw-r--r-- | src/include/optimizer/planmain.h | 2 |
15 files changed, 436 insertions, 112 deletions
diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 29f203c6f10..e8907709bd9 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -26,6 +26,8 @@ #include "nodes/makefuncs.h" #include "optimizer/cost.h" #include "optimizer/pathnode.h" +#include "optimizer/planmain.h" +#include "optimizer/restrictinfo.h" #include "utils/rel.h" PG_MODULE_MAGIC; @@ -48,7 +50,7 @@ struct FileFdwOption * Note: If you are adding new option for user mapping, you need to modify * fileGetOptions(), which currently doesn't bother to look at user mappings. */ -static struct FileFdwOption valid_options[] = { +static const struct FileFdwOption valid_options[] = { /* File options */ {"filename", ForeignTableRelationId}, @@ -72,6 +74,17 @@ static struct FileFdwOption valid_options[] = { }; /* + * FDW-specific information for RelOptInfo.fdw_private. + */ +typedef struct FileFdwPlanState +{ + char *filename; /* file to read */ + List *options; /* merged COPY options, excluding filename */ + BlockNumber pages; /* estimate of file's physical size */ + double ntuples; /* estimate of number of rows in file */ +} FileFdwPlanState; + +/* * FDW-specific information for ForeignScanState.fdw_state. */ typedef struct FileFdwExecutionState @@ -93,9 +106,18 @@ PG_FUNCTION_INFO_V1(file_fdw_validator); /* * FDW callback routines */ -static void filePlanForeignScan(Oid foreigntableid, - PlannerInfo *root, - RelOptInfo *baserel); +static void fileGetForeignRelSize(PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid); +static void fileGetForeignPaths(PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid); +static ForeignScan *fileGetForeignPlan(PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid, + ForeignPath *best_path, + List *tlist, + List *scan_clauses); static void fileExplainForeignScan(ForeignScanState *node, ExplainState *es); static void fileBeginForeignScan(ForeignScanState *node, int eflags); static TupleTableSlot *fileIterateForeignScan(ForeignScanState *node); @@ -109,8 +131,10 @@ static bool is_valid_option(const char *option, Oid context); static void fileGetOptions(Oid foreigntableid, char **filename, List **other_options); static List *get_file_fdw_attribute_options(Oid relid); +static void estimate_size(PlannerInfo *root, RelOptInfo *baserel, + FileFdwPlanState *fdw_private); static void estimate_costs(PlannerInfo *root, RelOptInfo *baserel, - const char *filename, + FileFdwPlanState *fdw_private, Cost *startup_cost, Cost *total_cost); @@ -123,7 +147,9 @@ file_fdw_handler(PG_FUNCTION_ARGS) { FdwRoutine *fdwroutine = makeNode(FdwRoutine); - fdwroutine->PlanForeignScan = filePlanForeignScan; + fdwroutine->GetForeignRelSize = fileGetForeignRelSize; + fdwroutine->GetForeignPaths = fileGetForeignPaths; + fdwroutine->GetForeignPlan = fileGetForeignPlan; fdwroutine->ExplainForeignScan = fileExplainForeignScan; fdwroutine->BeginForeignScan = fileBeginForeignScan; fdwroutine->IterateForeignScan = fileIterateForeignScan; @@ -177,7 +203,7 @@ file_fdw_validator(PG_FUNCTION_ARGS) if (!is_valid_option(def->defname, catalog)) { - struct FileFdwOption *opt; + const struct FileFdwOption *opt; StringInfoData buf; /* @@ -249,7 +275,7 @@ file_fdw_validator(PG_FUNCTION_ARGS) static bool is_valid_option(const char *option, Oid context) { - struct FileFdwOption *opt; + const struct FileFdwOption *opt; for (opt = valid_options; opt->optname; opt++) { @@ -381,7 +407,31 @@ get_file_fdw_attribute_options(Oid relid) } /* - * filePlanForeignScan + * fileGetForeignRelSize + * Obtain relation size estimates for a foreign table + */ +static void +fileGetForeignRelSize(PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid) +{ + FileFdwPlanState *fdw_private; + + /* + * Fetch options. We only need filename at this point, but we might + * as well get everything and not need to re-fetch it later in planning. + */ + fdw_private = (FileFdwPlanState *) palloc(sizeof(FileFdwPlanState)); + fileGetOptions(foreigntableid, + &fdw_private->filename, &fdw_private->options); + baserel->fdw_private = (void *) fdw_private; + + /* Estimate relation size */ + estimate_size(root, baserel, fdw_private); +} + +/* + * fileGetForeignPaths * Create possible access paths for a scan on the foreign table * * Currently we don't support any push-down feature, so there is only one @@ -389,20 +439,16 @@ get_file_fdw_attribute_options(Oid relid) * the data file. */ static void -filePlanForeignScan(Oid foreigntableid, - PlannerInfo *root, - RelOptInfo *baserel) +fileGetForeignPaths(PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid) { - char *filename; - List *options; + FileFdwPlanState *fdw_private = (FileFdwPlanState *) baserel->fdw_private; Cost startup_cost; Cost total_cost; - /* Fetch options --- we only need filename at this point */ - fileGetOptions(foreigntableid, &filename, &options); - - /* Estimate costs and update baserel->rows */ - estimate_costs(root, baserel, filename, + /* Estimate costs */ + estimate_costs(root, baserel, fdw_private, &startup_cost, &total_cost); /* Create a ForeignPath node and add it as only possible path */ @@ -423,6 +469,37 @@ filePlanForeignScan(Oid foreigntableid, } /* + * fileGetForeignPlan + * Create a ForeignScan plan node for scanning the foreign table + */ +static ForeignScan * +fileGetForeignPlan(PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid, + ForeignPath *best_path, + List *tlist, + List *scan_clauses) +{ + Index scan_relid = baserel->relid; + + /* + * We have no native ability to evaluate restriction clauses, so we just + * put all the scan_clauses into the plan node's qual list for the + * executor to check. So all we have to do here is strip RestrictInfo + * nodes from the clauses and ignore pseudoconstants (which will be + * handled elsewhere). + */ + scan_clauses = extract_actual_clauses(scan_clauses, false); + + /* Create the ForeignScan node */ + return make_foreignscan(tlist, + scan_clauses, + scan_relid, + NIL, /* no expressions to evaluate */ + NIL); /* no private state either */ +} + +/* * fileExplainForeignScan * Produce extra output for EXPLAIN */ @@ -568,38 +645,38 @@ fileReScanForeignScan(ForeignScanState *node) } /* - * Estimate costs of scanning a foreign table. + * Estimate size of a foreign table. * - * In addition to setting *startup_cost and *total_cost, this should - * update baserel->rows. + * The main result is returned in baserel->rows. We also set + * fdw_private->pages and fdw_private->ntuples for later use in the cost + * calculation. */ static void -estimate_costs(PlannerInfo *root, RelOptInfo *baserel, - const char *filename, - Cost *startup_cost, Cost *total_cost) +estimate_size(PlannerInfo *root, RelOptInfo *baserel, + FileFdwPlanState *fdw_private) { struct stat stat_buf; BlockNumber pages; int tuple_width; double ntuples; double nrows; - Cost run_cost = 0; - Cost cpu_per_tuple; /* * Get size of the file. It might not be there at plan time, though, in * which case we have to use a default estimate. */ - if (stat(filename, &stat_buf) < 0) + if (stat(fdw_private->filename, &stat_buf) < 0) stat_buf.st_size = 10 * BLCKSZ; /* - * Convert size to pages for use in I/O cost estimate below. + * Convert size to pages for use in I/O cost estimate later. */ pages = (stat_buf.st_size + (BLCKSZ - 1)) / BLCKSZ; if (pages < 1) pages = 1; + fdw_private->pages = pages; + /* * Estimate the number of tuples in the file. We back into this estimate * using the planner's idea of the relation width; which is bogus if not @@ -611,6 +688,8 @@ estimate_costs(PlannerInfo *root, RelOptInfo *baserel, ntuples = clamp_row_est((double) stat_buf.st_size / (double) tuple_width); + fdw_private->ntuples = ntuples; + /* * Now estimate the number of rows returned by the scan after applying the * baserestrictinfo quals. This is pretty bogus too, since the planner @@ -627,12 +706,28 @@ estimate_costs(PlannerInfo *root, RelOptInfo *baserel, /* Save the output-rows estimate for the planner */ baserel->rows = nrows; +} + +/* + * Estimate costs of scanning a foreign table. + * + * Results are returned in *startup_cost and *total_cost. + */ +static void +estimate_costs(PlannerInfo *root, RelOptInfo *baserel, + FileFdwPlanState *fdw_private, + Cost *startup_cost, Cost *total_cost) +{ + BlockNumber pages = fdw_private->pages; + double ntuples = fdw_private->ntuples; + Cost run_cost = 0; + Cost cpu_per_tuple; /* - * Now estimate costs. We estimate costs almost the same way as - * cost_seqscan(), thus assuming that I/O costs are equivalent to a - * regular table file of the same size. However, we take per-tuple CPU - * costs as 10x of a seqscan, to account for the cost of parsing records. + * We estimate costs almost the same way as cost_seqscan(), thus assuming + * that I/O costs are equivalent to a regular table file of the same size. + * However, we take per-tuple CPU costs as 10x of a seqscan, to account + * for the cost of parsing records. */ run_cost += seq_page_cost * pages; diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml index dbfcbbc2b36..f7bf3d8a395 100644 --- a/doc/src/sgml/fdwhandler.sgml +++ b/doc/src/sgml/fdwhandler.sgml @@ -89,52 +89,92 @@ <para> <programlisting> void -PlanForeignScan (Oid foreigntableid, - PlannerInfo *root, - RelOptInfo *baserel); +GetForeignRelSize (PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid); </programlisting> - Create possible access paths for a scan on a foreign table. This is - called when a query is planned. + Obtain relation size estimates for a foreign table. This is called + at the beginning of planning for a query involving a foreign table. + <literal>root</> is the planner's global information about the query; + <literal>baserel</> is the planner's information about this table; and <literal>foreigntableid</> is the <structname>pg_class</> OID of the - foreign table. <literal>root</> is the planner's global information - about the query, and <literal>baserel</> is the planner's information - about this table. + foreign table. (<literal>foreigntableid</> could be obtained from the + planner data structures, but it's passed explicitly to save effort.) </para> <para> - The function must generate at least one access path (ForeignPath node) - for a scan on the foreign table and must call <function>add_path</> to - add the path to <literal>baserel->pathlist</>. It's recommended to - use <function>create_foreignscan_path</> to build the ForeignPath node. - The function may generate multiple access paths, e.g., a path which has - valid <literal>pathkeys</> to represent a pre-sorted result. Each access - path must contain cost estimates, and can contain any FDW-private - information that is needed to execute the foreign scan at a later time. - (Note that the private information must be represented in a form that - <function>copyObject</> knows how to copy.) + This function should update <literal>baserel->rows</> to be the + expected number of rows returned by the table scan, after accounting for + the filtering done by the restriction quals. The initial value of + <literal>baserel->rows</> is just a constant default estimate, which + should be replaced if at all possible. The function may also choose to + update <literal>baserel->width</> if it can compute a better estimate + of the average result row width. </para> <para> - The information in <literal>root</> and <literal>baserel</> can be used - to reduce the amount of information that has to be fetched from the - foreign table (and therefore reduce the cost estimate). - <literal>baserel->baserestrictinfo</> is particularly interesting, as - it contains restriction quals (<literal>WHERE</> clauses) that can be - used to filter the rows to be fetched. (The FDW is not required to - enforce these quals, as the finished plan will recheck them anyway.) - <literal>baserel->reltargetlist</> can be used to determine which - columns need to be fetched. + See <xref linkend="fdw-planning"> for additional information. + </para> + + <para> +<programlisting> +void +GetForeignPaths (PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid); +</programlisting> + + Create possible access paths for a scan on a foreign table. + This is called during query planning. + The parameters are the same as for <function>GetForeignRelSize</>, + which has already been called. + </para> + + <para> + This function must generate at least one access path + (<structname>ForeignPath</> node) for a scan on the foreign table and + must call <function>add_path</> to add each such path to + <literal>baserel->pathlist</>. It's recommended to use + <function>create_foreignscan_path</> to build the + <structname>ForeignPath</> nodes. The function can generate multiple + access paths, e.g., a path which has valid <literal>pathkeys</> to + represent a pre-sorted result. Each access path must contain cost + estimates, and can contain any FDW-private information that is needed to + identify the specific scan method intended. + </para> + + <para> + See <xref linkend="fdw-planning"> for additional information. + </para> + + <para> +<programlisting> +ForeignScan * +GetForeignPlan (PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid, + ForeignPath *best_path, + List *tlist, + List *scan_clauses); +</programlisting> + + Create a <structname>ForeignScan</> plan node from the selected foreign + access path. This is called at the end of query planning. + The parameters are as for <function>GetForeignRelSize</>, plus + the selected <structname>ForeignPath</> (previously produced by + <function>GetForeignPaths</>), the target list to be emitted by the + plan node, and the restriction clauses to be enforced by the plan node. </para> <para> - In addition to returning cost estimates, the function should update - <literal>baserel->rows</> to be the expected number of rows returned - by the scan, after accounting for the filtering done by the restriction - quals. The initial value of <literal>baserel->rows</> is just a - constant default estimate, which should be replaced if at all possible. - The function may also choose to update <literal>baserel->width</> if - it can compute a better estimate of the average result row width. + This function must create and return a <structname>ForeignScan</> plan + node; it's recommended to use <function>make_foreignscan</> to build the + <structname>ForeignScan</> node. + </para> + + <para> + See <xref linkend="fdw-planning"> for additional information. </para> <para> @@ -170,7 +210,7 @@ BeginForeignScan (ForeignScanState *node, the table to scan is accessible through the <structname>ForeignScanState</> node (in particular, from the underlying <structname>ForeignScan</> plan node, which contains any FDW-private - information provided by <function>PlanForeignScan</>). + information provided by <function>GetForeignPlan</>). </para> <para> @@ -347,6 +387,126 @@ GetForeignServerByName(const char *name, bool missing_ok); return NULL if missing_ok is true, otherwise raise an error. </para> + </sect1> + + <sect1 id="fdw-planning"> + <title>Foreign Data Wrapper Query Planning</title> + + <para> + The FDW callback functions <function>GetForeignRelSize</>, + <function>GetForeignPaths</>, and <function>GetForeignPlan</> must fit + into the workings of the <productname>PostgreSQL</> planner. Here are + some notes about what they must do. + </para> + + <para> + The information in <literal>root</> and <literal>baserel</> can be used + to reduce the amount of information that has to be fetched from the + foreign table (and therefore reduce the cost). + <literal>baserel->baserestrictinfo</> is particularly interesting, as + it contains restriction quals (<literal>WHERE</> clauses) that should be + used to filter the rows to be fetched. (The FDW itself is not required + to enforce these quals, as the core executor can check them instead.) + <literal>baserel->reltargetlist</> can be used to determine which + columns need to be fetched; but note that it only lists columns that + have to be emitted by the <structname>ForeignScan</> plan node, not + columns that are used in qual evaluation but not output by the query. + </para> + + <para> + Various private fields are available for the FDW planning functions to + keep information in. Generally, whatever you store in FDW private fields + should be palloc'd, so that it will be reclaimed at the end of planning. + </para> + + <para> + <literal>baserel->fdw_private</> is a <type>void</> pointer that is + available for FDW planning functions to store information relevant to + the particular foreign table. The core planner does not touch it except + to initialize it to NULL when the <literal>baserel</> node is created. + It is useful for passing information forward from + <function>GetForeignRelSize</> to <function>GetForeignPaths</> and/or + <function>GetForeignPaths</> to <function>GetForeignPlan</>, thereby + avoiding recalculation. + </para> + + <para> + <function>GetForeignPaths</> can identify the meaning of different + access paths by storing private information in the + <structfield>fdw_private</> field of <structname>ForeignPath</> nodes. + <structfield>fdw_private</> is declared as a <type>List</> pointer, but + could actually contain anything since the core planner does not touch + it. However, best practice is to use a representation that's dumpable + by <function>nodeToString</>, for use with debugging support available + in the backend. + </para> + + <para> + <function>GetForeignPlan</> can examine the <structfield>fdw_private</> + field of the selected <structname>ForeignPath</> node, and can generate + <structfield>fdw_exprs</> and <structfield>fdw_private</> lists to be + placed in the <structname>ForeignScan</> plan node, where they will be + available at execution time. Both of these lists must be + represented in a form that <function>copyObject</> knows how to copy. + The <structfield>fdw_private</> list has no other restrictions and is + not interpreted by the core backend in any way. The + <structfield>fdw_exprs</> list, if not NIL, is expected to contain + expression trees that are intended to be executed at runtime. These + trees will undergo post-processing by the planner to make them fully + executable. + </para> + + <para> + In <function>GetForeignPlan</>, generally the passed-in targetlist can + be copied into the plan node as-is. The passed scan_clauses list + contains the same clauses as <literal>baserel->baserestrictinfo</>, + but may be re-ordered for better execution efficiency. In simple cases + the FDW can just strip <structname>RestrictInfo</> nodes from the + scan_clauses list (using <function>extract_actual_clauses</>) and put + all the clauses into the plan node's qual list, which means that all the + clauses will be checked by the executor at runtime. More complex FDWs + may be able to check some of the clauses internally, in which case those + clauses can be removed from the plan node's qual list so that the + executor doesn't waste time rechecking them. + </para> + + <para> + As an example, the FDW might identify some restriction clauses of the + form <replaceable>foreign_variable</> <literal>=</> + <replaceable>sub_expression</>, which it determines can be executed on + the remote server given the locally-evaluated value of the + <replaceable>sub_expression</>. The actual identification of such a + clause should happen during <function>GetForeignPaths</>, since it would + affect the cost estimate for the path. The path's + <structfield>fdw_private</> field would probably include a pointer to + the identified clause's <structname>RestrictInfo</> node. Then + <function>GetForeignPlan</> would remove that clause from scan_clauses, + but add the <replaceable>sub_expression</> to <structfield>fdw_exprs</> + to ensure that it gets massaged into executable form. It would probably + also put control information into the plan node's + <structfield>fdw_private</> field to tell the execution functions what + to do at runtime. The query transmitted to the remote server would + involve something like <literal>WHERE <replaceable>foreign_variable</> = + $1</literal>, with the parameter value obtained at runtime from + evaluation of the <structfield>fdw_exprs</> expression tree. + </para> + + <para> + The FDW should always construct at least one path that depends only on + the table's restriction clauses. In join queries, it might also choose + to construct path(s) that depend on join clauses, for example + <replaceable>foreign_variable</> <literal>=</> + <replaceable>local_variable</>. Such clauses will not be found in + <literal>baserel->baserestrictinfo</> but must be sought in the + relation's join lists. A path using such a clause is called a + <quote>parameterized path</>. It must show the other relation(s) as + <literal>required_outer</> and list the specific join clause(s) in + <literal>param_clauses</>. In <function>GetForeignPlan</>, the + <replaceable>local_variable</> portion of the join clause would be added + to <structfield>fdw_exprs</>, and then at runtime the case works the + same as for an ordinary restriction clause. + </para> + </sect1> </chapter> diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 868fb7130a8..5cde22543f5 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -591,8 +591,9 @@ _copyForeignScan(const ForeignScan *from) /* * copy remainder of node */ - COPY_SCALAR_FIELD(fsSystemCol); + COPY_NODE_FIELD(fdw_exprs); COPY_NODE_FIELD(fdw_private); + COPY_SCALAR_FIELD(fsSystemCol); return newnode; } diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 9daeb3e7b43..51181a9a743 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -559,8 +559,9 @@ _outForeignScan(StringInfo str, const ForeignScan *node) _outScanInfo(str, (const Scan *) node); - WRITE_BOOL_FIELD(fsSystemCol); + WRITE_NODE_FIELD(fdw_exprs); WRITE_NODE_FIELD(fdw_private); + WRITE_BOOL_FIELD(fsSystemCol); } static void @@ -1741,6 +1742,7 @@ _outRelOptInfo(StringInfo str, const RelOptInfo *node) WRITE_FLOAT_FIELD(allvisfrac, "%.6f"); WRITE_NODE_FIELD(subplan); WRITE_NODE_FIELD(subroot); + /* we don't try to print fdwroutine or fdw_private */ WRITE_NODE_FIELD(baserestrictinfo); WRITE_NODE_FIELD(joininfo); WRITE_BOOL_FIELD(has_eclass_joins); diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 6e81ce0fc26..03c604a03d6 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -396,6 +396,12 @@ set_foreign_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { /* Mark rel with estimated output rows, width, etc */ set_foreign_size_estimates(root, rel); + + /* Get FDW routine pointers for the rel */ + rel->fdwroutine = GetFdwRoutineByRelId(rte->relid); + + /* Let FDW adjust the size estimates, if it can */ + rel->fdwroutine->GetForeignRelSize(root, rel, rte->relid); } /* @@ -405,11 +411,8 @@ set_foreign_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) static void set_foreign_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { - FdwRoutine *fdwroutine; - - /* Call the FDW's PlanForeignScan function to generate path(s) */ - fdwroutine = GetFdwRoutineByRelId(rte->relid); - fdwroutine->PlanForeignScan(rte->relid, root, rel); + /* Call the FDW's GetForeignPaths function to generate path(s) */ + rel->fdwroutine->GetForeignPaths(root, rel, rte->relid); /* Select cheapest path */ set_cheapest(rel); diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 885d8558c31..24c853d47ef 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -3745,7 +3745,7 @@ set_cte_size_estimates(PlannerInfo *root, RelOptInfo *rel, Plan *cteplan) * using what will be purely datatype-driven estimates from the targetlist. * There is no way to do anything sane with the rows value, so we just put * a default estimate and hope that the wrapper can improve on it. The - * wrapper's PlanForeignScan function will be called momentarily. + * wrapper's GetForeignRelSize function will be called momentarily. * * The rel's targetlist and restrictinfo list must have been constructed * already. diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index b1df56cafd2..94140d304f7 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -20,6 +20,7 @@ #include <math.h> #include "access/skey.h" +#include "foreign/fdwapi.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" @@ -119,8 +120,6 @@ static CteScan *make_ctescan(List *qptlist, List *qpqual, Index scanrelid, int ctePlanId, int cteParam); static WorkTableScan *make_worktablescan(List *qptlist, List *qpqual, Index scanrelid, int wtParam); -static ForeignScan *make_foreignscan(List *qptlist, List *qpqual, - Index scanrelid, bool fsSystemCol, List *fdw_private); static BitmapAnd *make_bitmap_and(List *bitmapplans); static BitmapOr *make_bitmap_or(List *bitmapplans); static NestLoop *make_nestloop(List *tlist, @@ -1816,7 +1815,6 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path, RelOptInfo *rel = best_path->path.parent; Index scan_relid = rel->relid; RangeTblEntry *rte; - bool fsSystemCol; int i; /* it should be a base rel... */ @@ -1825,31 +1823,56 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path, rte = planner_rt_fetch(scan_relid, root); Assert(rte->rtekind == RTE_RELATION); - /* Sort clauses into best execution order */ + /* + * Sort clauses into best execution order. We do this first since the + * FDW might have more info than we do and wish to adjust the ordering. + */ scan_clauses = order_qual_clauses(root, scan_clauses); - /* Reduce RestrictInfo list to bare expressions; ignore pseudoconstants */ - scan_clauses = extract_actual_clauses(scan_clauses, false); + /* + * Let the FDW perform its processing on the restriction clauses and + * generate the plan node. Note that the FDW might remove restriction + * clauses that it intends to execute remotely, or even add more (if it + * has selected some join clauses for remote use but also wants them + * rechecked locally). + */ + scan_plan = rel->fdwroutine->GetForeignPlan(root, rel, rte->relid, + best_path, + tlist, scan_clauses); + + /* Copy cost data from Path to Plan; no need to make FDW do this */ + copy_path_costsize(&scan_plan->scan.plan, &best_path->path); - /* Detect whether any system columns are requested from rel */ - fsSystemCol = false; + /* + * Replace any outer-relation variables with nestloop params in the qual + * and fdw_exprs expressions. We do this last so that the FDW doesn't + * have to be involved. (Note that parts of fdw_exprs could have come + * from join clauses, so doing this beforehand on the scan_clauses + * wouldn't work.) + */ + if (best_path->path.required_outer) + { + scan_plan->scan.plan.qual = (List *) + replace_nestloop_params(root, (Node *) scan_plan->scan.plan.qual); + scan_plan->fdw_exprs = (List *) + replace_nestloop_params(root, (Node *) scan_plan->fdw_exprs); + } + + /* + * Detect whether any system columns are requested from rel. This is a + * bit of a kluge and might go away someday, so we intentionally leave it + * out of the API presented to FDWs. + */ + scan_plan->fsSystemCol = false; for (i = rel->min_attr; i < 0; i++) { if (!bms_is_empty(rel->attr_needed[i - rel->min_attr])) { - fsSystemCol = true; + scan_plan->fsSystemCol = true; break; } } - scan_plan = make_foreignscan(tlist, - scan_clauses, - scan_relid, - fsSystemCol, - best_path->fdw_private); - - copy_path_costsize(&scan_plan->scan.plan, &best_path->path); - return scan_plan; } @@ -3183,24 +3206,26 @@ make_worktablescan(List *qptlist, return node; } -static ForeignScan * +ForeignScan * make_foreignscan(List *qptlist, List *qpqual, Index scanrelid, - bool fsSystemCol, + List *fdw_exprs, List *fdw_private) { ForeignScan *node = makeNode(ForeignScan); Plan *plan = &node->scan.plan; - /* cost should be inserted by caller */ + /* cost will be filled in by create_foreignscan_plan */ plan->targetlist = qptlist; plan->qual = qpqual; plan->lefttree = NULL; plan->righttree = NULL; node->scan.scanrelid = scanrelid; - node->fsSystemCol = fsSystemCol; + node->fdw_exprs = fdw_exprs; node->fdw_private = fdw_private; + /* fsSystemCol will be filled in by create_foreignscan_plan */ + node->fsSystemCol = false; return node; } diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index e1b48fb4f53..69396694aaa 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -428,6 +428,8 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); splan->scan.plan.qual = fix_scan_list(root, splan->scan.plan.qual, rtoffset); + splan->fdw_exprs = + fix_scan_list(root, splan->fdw_exprs, rtoffset); } break; diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index 40a420a3546..b64db1e1c06 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -2137,6 +2137,8 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params, break; case T_ForeignScan: + finalize_primnode((Node *) ((ForeignScan *) plan)->fdw_exprs, + &context); context.paramids = bms_add_members(context.paramids, scan_params); break; diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 6d1545476df..a2fc75a659e 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1767,7 +1767,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel) * returning the pathnode. * * This function is never called from core Postgres; rather, it's expected - * to be called by the PlanForeignScan function of a foreign data wrapper. + * to be called by the GetForeignPaths function of a foreign data wrapper. * We make the FDW supply all fields of the path, since we do not have any * way to calculate them in core. */ diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 0cdf638c1dd..cee092a8810 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -113,6 +113,8 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptKind reloptkind) rel->allvisfrac = 0; rel->subplan = NULL; rel->subroot = NULL; + rel->fdwroutine = NULL; + rel->fdw_private = NULL; rel->baserestrictinfo = NIL; rel->baserestrictcost.startup = 0; rel->baserestrictcost.per_tuple = 0; @@ -366,6 +368,8 @@ build_join_rel(PlannerInfo *root, joinrel->allvisfrac = 0; joinrel->subplan = NULL; joinrel->subroot = NULL; + joinrel->fdwroutine = NULL; + joinrel->fdw_private = NULL; joinrel->baserestrictinfo = NIL; joinrel->baserestrictcost.startup = 0; joinrel->baserestrictcost.per_tuple = 0; diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h index 9e135c62069..854f17755c4 100644 --- a/src/include/foreign/fdwapi.h +++ b/src/include/foreign/fdwapi.h @@ -23,9 +23,20 @@ struct ExplainState; * Callback function signatures --- see fdwhandler.sgml for more info. */ -typedef void (*PlanForeignScan_function) (Oid foreigntableid, - PlannerInfo *root, - RelOptInfo *baserel); +typedef void (*GetForeignRelSize_function) (PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid); + +typedef void (*GetForeignPaths_function) (PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid); + +typedef ForeignScan *(*GetForeignPlan_function) (PlannerInfo *root, + RelOptInfo *baserel, + Oid foreigntableid, + ForeignPath *best_path, + List *tlist, + List *scan_clauses); typedef void (*ExplainForeignScan_function) (ForeignScanState *node, struct ExplainState *es); @@ -53,7 +64,9 @@ typedef struct FdwRoutine { NodeTag type; - PlanForeignScan_function PlanForeignScan; + GetForeignRelSize_function GetForeignRelSize; + GetForeignPaths_function GetForeignPaths; + GetForeignPlan_function GetForeignPlan; ExplainForeignScan_function ExplainForeignScan; BeginForeignScan_function BeginForeignScan; IterateForeignScan_function IterateForeignScan; diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 3962792d3d8..e6bb3239f42 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -462,13 +462,22 @@ typedef struct WorkTableScan /* ---------------- * ForeignScan node + * + * fdw_exprs and fdw_private are both under the control of the foreign-data + * wrapper, but fdw_exprs is presumed to contain expression trees and will + * be post-processed accordingly by the planner; fdw_private won't be. + * Note that everything in both lists must be copiable by copyObject(). + * One way to store an arbitrary blob of bytes is to represent it as a bytea + * Const. Usually, though, you'll be better off choosing a representation + * that can be dumped usefully by nodeToString(). * ---------------- */ typedef struct ForeignScan { Scan scan; - bool fsSystemCol; /* true if any "system column" is needed */ + List *fdw_exprs; /* expressions that FDW may evaluate */ List *fdw_private; /* private data for FDW */ + bool fsSystemCol; /* true if any "system column" is needed */ } ForeignScan; diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 2a686080059..8616223f24a 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -334,10 +334,13 @@ typedef struct PlannerInfo * allvisfrac - fraction of disk pages that are marked all-visible * subplan - plan for subquery (NULL if it's not a subquery) * subroot - PlannerInfo for subquery (NULL if it's not a subquery) + * fdwroutine - function hooks for FDW, if foreign table (else NULL) + * fdw_private - private state for FDW, if foreign table (else NULL) * * Note: for a subquery, tuples, subplan, subroot are not set immediately * upon creation of the RelOptInfo object; they are filled in when - * set_base_rel_pathlist processes the object. + * set_subquery_pathlist processes the object. Likewise, fdwroutine + * and fdw_private are filled during initial path creation. * * For otherrels that are appendrel members, these fields are filled * in just as for a baserel. @@ -414,8 +417,12 @@ typedef struct RelOptInfo BlockNumber pages; /* size estimates derived from pg_class */ double tuples; double allvisfrac; + /* use "struct Plan" to avoid including plannodes.h here */ struct Plan *subplan; /* if subquery */ PlannerInfo *subroot; /* if subquery */ + /* use "struct FdwRoutine" to avoid including fdwapi.h here */ + struct FdwRoutine *fdwroutine; /* if foreign table */ + void *fdw_private; /* if foreign table */ /* used by various scans and joins: */ List *baserestrictinfo; /* RestrictInfo structures (if base @@ -793,14 +800,13 @@ typedef struct TidPath } TidPath; /* - * ForeignPath represents a scan of a foreign table - * - * fdw_private contains FDW private data about the scan, which will be copied - * to the final ForeignScan plan node so that it is available at execution - * time. Note that everything in this list must be copiable by copyObject(). - * One way to store an arbitrary blob of bytes is to represent it as a bytea - * Const. Usually, though, you'll be better off choosing a representation - * that can be dumped usefully by nodeToString(). + * ForeignPath represents a potential scan of a foreign table + * + * fdw_private stores FDW private data about the scan. While fdw_private is + * not actually touched by the core code during normal operations, it's + * generally a good idea to use a representation that can be dumped by + * nodeToString(), so that you can examine the structure during debugging + * with tools like pprint(). */ typedef struct ForeignPath { diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index 8bd603124b3..47cc39cf1d9 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -42,6 +42,8 @@ extern Plan *optimize_minmax_aggregates(PlannerInfo *root, List *tlist, extern Plan *create_plan(PlannerInfo *root, Path *best_path); extern SubqueryScan *make_subqueryscan(List *qptlist, List *qpqual, Index scanrelid, Plan *subplan); +extern ForeignScan *make_foreignscan(List *qptlist, List *qpqual, + Index scanrelid, List *fdw_exprs, List *fdw_private); extern Append *make_append(List *appendplans, List *tlist); extern RecursiveUnion *make_recursive_union(List *tlist, Plan *lefttree, Plan *righttree, int wtParam, |