Fix planner problems with LATERAL references in PlaceHolderVars.

The planner largely failed to consider the possibility that a
PlaceHolderVar's expression might contain a lateral reference to a Var
coming from somewhere outside the PHV's syntactic scope.  We had a previous
report of a problem in this area, which I tried to fix in a quick-hack way
in commit 4da6439bd8553059766011e2a42c6e39df08717f, but Antonin Houska
pointed out that there were still some problems, and investigation turned
up other issues.  This patch largely reverts that commit in favor of a more
thoroughly thought-through solution.  The new theory is that a PHV's
ph_eval_at level cannot be higher than its original syntactic level.  If it
contains lateral references, those don't change the ph_eval_at level, but
rather they create a lateral-reference requirement for the ph_eval_at join
relation.  The code in joinpath.c needs to handle that.

Another issue is that createplan.c wasn't handling nested PlaceHolderVars
properly.

In passing, push knowledge of lateral-reference checks for join clauses
into join_clause_is_movable_to.  This is mainly so that FDWs don't need
to deal with it.

This patch doesn't fix the original join-qual-placement problem reported by
Jeremy Evans (and indeed, one of the new regression test cases shows the
wrong answer because of that).  But the PlaceHolderVar problems need to be
fixed before that issue can be addressed, so committing this separately
seems reasonable.

1 files changed, 92 insertions, 11 deletions

diff --git a/src/backend/optimizer/path/joinpath.c b/src/backend/optimizer/path/joinpath.c
index d6050a616c7..5b477e52d3f 100644
--- a/src/backend/optimizer/path/joinpath.c
+++ b/src/backend/optimizer/path/joinpath.c
@@ -29,19 +29,19 @@ static void sort_inner_and_outer(PlannerInfo *root, RelOptInfo *joinrel,
 					 RelOptInfo *outerrel, RelOptInfo *innerrel,
 					 List *restrictlist, List *mergeclause_list,
 					 JoinType jointype, SpecialJoinInfo *sjinfo,
-					 Relids param_source_rels);
+					 Relids param_source_rels, Relids extra_lateral_rels);
 static void match_unsorted_outer(PlannerInfo *root, RelOptInfo *joinrel,
 					 RelOptInfo *outerrel, RelOptInfo *innerrel,
 					 List *restrictlist, List *mergeclause_list,
 					 JoinType jointype, SpecialJoinInfo *sjinfo,
 					 SemiAntiJoinFactors *semifactors,
-					 Relids param_source_rels);
+					 Relids param_source_rels, Relids extra_lateral_rels);
 static void hash_inner_and_outer(PlannerInfo *root, RelOptInfo *joinrel,
 					 RelOptInfo *outerrel, RelOptInfo *innerrel,
 					 List *restrictlist,
 					 JoinType jointype, SpecialJoinInfo *sjinfo,
 					 SemiAntiJoinFactors *semifactors,
-					 Relids param_source_rels);
+					 Relids param_source_rels, Relids extra_lateral_rels);
 static List *select_mergejoin_clauses(PlannerInfo *root,
 						 RelOptInfo *joinrel,
 						 RelOptInfo *outerrel,
@@ -87,6 +87,7 @@ add_paths_to_joinrel(PlannerInfo *root,
 	bool		mergejoin_allowed = true;
 	SemiAntiJoinFactors semifactors;
 	Relids		param_source_rels = NULL;
+	Relids		extra_lateral_rels = NULL;
 	ListCell   *lc;
 
 	/*
@@ -162,20 +163,58 @@ add_paths_to_joinrel(PlannerInfo *root,
 	{
 		LateralJoinInfo *ljinfo = (LateralJoinInfo *) lfirst(lc);
 
-		if (bms_is_member(ljinfo->lateral_rhs, joinrel->relids))
+		if (bms_is_subset(ljinfo->lateral_rhs, joinrel->relids))
 			param_source_rels = bms_join(param_source_rels,
 										 bms_difference(ljinfo->lateral_lhs,
 														joinrel->relids));
 	}
 
 	/*
+	 * Another issue created by LATERAL references is that PlaceHolderVars
+	 * that need to be computed at this join level might contain lateral
+	 * references to rels not in the join, meaning that the paths for the join
+	 * would need to be marked as parameterized by those rels, independently
+	 * of all other considerations.  Set extra_lateral_rels to the set of such
+	 * rels.  This will not affect our decisions as to which paths to
+	 * generate; we merely add these rels to their required_outer sets.
+	 */
+	foreach(lc, root->placeholder_list)
+	{
+		PlaceHolderInfo *phinfo = (PlaceHolderInfo *) lfirst(lc);
+
+		/* PHVs without lateral refs can be skipped over quickly */
+		if (phinfo->ph_lateral == NULL)
+			continue;
+		/* Is it due to be evaluated at this join, and not in either input? */
+		if (bms_is_subset(phinfo->ph_eval_at, joinrel->relids) &&
+			!bms_is_subset(phinfo->ph_eval_at, outerrel->relids) &&
+			!bms_is_subset(phinfo->ph_eval_at, innerrel->relids))
+		{
+			/* Yes, remember its lateral rels */
+			extra_lateral_rels = bms_add_members(extra_lateral_rels,
+												 phinfo->ph_lateral);
+		}
+	}
+
+	/*
+	 * Make sure extra_lateral_rels doesn't list anything within the join, and
+	 * that it's NULL if empty.  (This allows us to use bms_add_members to add
+	 * it to required_outer below, while preserving the property that
+	 * required_outer is exactly NULL if empty.)
+	 */
+	extra_lateral_rels = bms_del_members(extra_lateral_rels, joinrel->relids);
+	if (bms_is_empty(extra_lateral_rels))
+		extra_lateral_rels = NULL;
+
+	/*
 	 * 1. Consider mergejoin paths where both relations must be explicitly
 	 * sorted.	Skip this if we can't mergejoin.
 	 */
 	if (mergejoin_allowed)
 		sort_inner_and_outer(root, joinrel, outerrel, innerrel,
 							 restrictlist, mergeclause_list, jointype,
-							 sjinfo, param_source_rels);
+							 sjinfo,
+							 param_source_rels, extra_lateral_rels);
 
 	/*
 	 * 2. Consider paths where the outer relation need not be explicitly
@@ -187,7 +226,8 @@ add_paths_to_joinrel(PlannerInfo *root,
 	if (mergejoin_allowed)
 		match_unsorted_outer(root, joinrel, outerrel, innerrel,
 							 restrictlist, mergeclause_list, jointype,
-							 sjinfo, &semifactors, param_source_rels);
+							 sjinfo, &semifactors,
+							 param_source_rels, extra_lateral_rels);
 
 #ifdef NOT_USED
 
@@ -205,7 +245,8 @@ add_paths_to_joinrel(PlannerInfo *root,
 	if (mergejoin_allowed)
 		match_unsorted_inner(root, joinrel, outerrel, innerrel,
 							 restrictlist, mergeclause_list, jointype,
-							 sjinfo, &semifactors, param_source_rels);
+							 sjinfo, &semifactors,
+							 param_source_rels, extra_lateral_rels);
 #endif
 
 	/*
@@ -216,7 +257,8 @@ add_paths_to_joinrel(PlannerInfo *root,
 	if (enable_hashjoin || jointype == JOIN_FULL)
 		hash_inner_and_outer(root, joinrel, outerrel, innerrel,
 							 restrictlist, jointype,
-							 sjinfo, &semifactors, param_source_rels);
+							 sjinfo, &semifactors,
+							 param_source_rels, extra_lateral_rels);
 }
 
 /*
@@ -231,6 +273,7 @@ try_nestloop_path(PlannerInfo *root,
 				  SpecialJoinInfo *sjinfo,
 				  SemiAntiJoinFactors *semifactors,
 				  Relids param_source_rels,
+				  Relids extra_lateral_rels,
 				  Path *outer_path,
 				  Path *inner_path,
 				  List *restrict_clauses,
@@ -254,6 +297,12 @@ try_nestloop_path(PlannerInfo *root,
 	}
 
 	/*
+	 * Independently of that, add parameterization needed for any
+	 * PlaceHolderVars that need to be computed at the join.
+	 */
+	required_outer = bms_add_members(required_outer, extra_lateral_rels);
+
+	/*
 	 * Do a precheck to quickly eliminate obviously-inferior paths.  We
 	 * calculate a cheap lower bound on the path's cost and then use
 	 * add_path_precheck() to see if the path is clearly going to be dominated
@@ -301,6 +350,7 @@ try_mergejoin_path(PlannerInfo *root,
 				   JoinType jointype,
 				   SpecialJoinInfo *sjinfo,
 				   Relids param_source_rels,
+				   Relids extra_lateral_rels,
 				   Path *outer_path,
 				   Path *inner_path,
 				   List *restrict_clauses,
@@ -327,6 +377,12 @@ try_mergejoin_path(PlannerInfo *root,
 	}
 
 	/*
+	 * Independently of that, add parameterization needed for any
+	 * PlaceHolderVars that need to be computed at the join.
+	 */
+	required_outer = bms_add_members(required_outer, extra_lateral_rels);
+
+	/*
 	 * If the given paths are already well enough ordered, we can skip doing
 	 * an explicit sort.
 	 */
@@ -383,6 +439,7 @@ try_hashjoin_path(PlannerInfo *root,
 				  SpecialJoinInfo *sjinfo,
 				  SemiAntiJoinFactors *semifactors,
 				  Relids param_source_rels,
+				  Relids extra_lateral_rels,
 				  Path *outer_path,
 				  Path *inner_path,
 				  List *restrict_clauses,
@@ -406,6 +463,12 @@ try_hashjoin_path(PlannerInfo *root,
 	}
 
 	/*
+	 * Independently of that, add parameterization needed for any
+	 * PlaceHolderVars that need to be computed at the join.
+	 */
+	required_outer = bms_add_members(required_outer, extra_lateral_rels);
+
+	/*
 	 * See comments in try_nestloop_path().  Also note that hashjoin paths
 	 * never have any output pathkeys, per comments in create_hashjoin_path.
 	 */
@@ -483,6 +546,7 @@ clause_sides_match_join(RestrictInfo *rinfo, RelOptInfo *outerrel,
  * 'jointype' is the type of join to do
  * 'sjinfo' is extra info about the join for selectivity estimation
  * 'param_source_rels' are OK targets for parameterization of result paths
+ * 'extra_lateral_rels' are additional parameterization for result paths
  */
 static void
 sort_inner_and_outer(PlannerInfo *root,
@@ -493,7 +557,8 @@ sort_inner_and_outer(PlannerInfo *root,
 					 List *mergeclause_list,
 					 JoinType jointype,
 					 SpecialJoinInfo *sjinfo,
-					 Relids param_source_rels)
+					 Relids param_source_rels,
+					 Relids extra_lateral_rels)
 {
 	Path	   *outer_path;
 	Path	   *inner_path;
@@ -623,6 +688,7 @@ sort_inner_and_outer(PlannerInfo *root,
 						   jointype,
 						   sjinfo,
 						   param_source_rels,
+						   extra_lateral_rels,
 						   outer_path,
 						   inner_path,
 						   restrictlist,
@@ -668,6 +734,7 @@ sort_inner_and_outer(PlannerInfo *root,
  * 'sjinfo' is extra info about the join for selectivity estimation
  * 'semifactors' contains valid data if jointype is SEMI or ANTI
  * 'param_source_rels' are OK targets for parameterization of result paths
+ * 'extra_lateral_rels' are additional parameterization for result paths
  */
 static void
 match_unsorted_outer(PlannerInfo *root,
@@ -679,7 +746,8 @@ match_unsorted_outer(PlannerInfo *root,
 					 JoinType jointype,
 					 SpecialJoinInfo *sjinfo,
 					 SemiAntiJoinFactors *semifactors,
-					 Relids param_source_rels)
+					 Relids param_source_rels,
+					 Relids extra_lateral_rels)
 {
 	JoinType	save_jointype = jointype;
 	bool		nestjoinOK;
@@ -809,6 +877,7 @@ match_unsorted_outer(PlannerInfo *root,
 							  sjinfo,
 							  semifactors,
 							  param_source_rels,
+							  extra_lateral_rels,
 							  outerpath,
 							  inner_cheapest_total,
 							  restrictlist,
@@ -834,6 +903,7 @@ match_unsorted_outer(PlannerInfo *root,
 								  sjinfo,
 								  semifactors,
 								  param_source_rels,
+								  extra_lateral_rels,
 								  outerpath,
 								  innerpath,
 								  restrictlist,
@@ -848,6 +918,7 @@ match_unsorted_outer(PlannerInfo *root,
 								  sjinfo,
 								  semifactors,
 								  param_source_rels,
+								  extra_lateral_rels,
 								  outerpath,
 								  matpath,
 								  restrictlist,
@@ -903,6 +974,7 @@ match_unsorted_outer(PlannerInfo *root,
 						   jointype,
 						   sjinfo,
 						   param_source_rels,
+						   extra_lateral_rels,
 						   outerpath,
 						   inner_cheapest_total,
 						   restrictlist,
@@ -1001,6 +1073,7 @@ match_unsorted_outer(PlannerInfo *root,
 								   jointype,
 								   sjinfo,
 								   param_source_rels,
+								   extra_lateral_rels,
 								   outerpath,
 								   innerpath,
 								   restrictlist,
@@ -1046,6 +1119,7 @@ match_unsorted_outer(PlannerInfo *root,
 									   jointype,
 									   sjinfo,
 									   param_source_rels,
+									   extra_lateral_rels,
 									   outerpath,
 									   innerpath,
 									   restrictlist,
@@ -1080,6 +1154,7 @@ match_unsorted_outer(PlannerInfo *root,
  * 'sjinfo' is extra info about the join for selectivity estimation
  * 'semifactors' contains valid data if jointype is SEMI or ANTI
  * 'param_source_rels' are OK targets for parameterization of result paths
+ * 'extra_lateral_rels' are additional parameterization for result paths
  */
 static void
 hash_inner_and_outer(PlannerInfo *root,
@@ -1090,7 +1165,8 @@ hash_inner_and_outer(PlannerInfo *root,
 					 JoinType jointype,
 					 SpecialJoinInfo *sjinfo,
 					 SemiAntiJoinFactors *semifactors,
-					 Relids param_source_rels)
+					 Relids param_source_rels,
+					 Relids extra_lateral_rels)
 {
 	bool		isouterjoin = IS_OUTER_JOIN(jointype);
 	List	   *hashclauses;
@@ -1164,6 +1240,7 @@ hash_inner_and_outer(PlannerInfo *root,
 							  sjinfo,
 							  semifactors,
 							  param_source_rels,
+							  extra_lateral_rels,
 							  cheapest_total_outer,
 							  cheapest_total_inner,
 							  restrictlist,
@@ -1183,6 +1260,7 @@ hash_inner_and_outer(PlannerInfo *root,
 							  sjinfo,
 							  semifactors,
 							  param_source_rels,
+							  extra_lateral_rels,
 							  cheapest_total_outer,
 							  cheapest_total_inner,
 							  restrictlist,
@@ -1195,6 +1273,7 @@ hash_inner_and_outer(PlannerInfo *root,
 								  sjinfo,
 								  semifactors,
 								  param_source_rels,
+								  extra_lateral_rels,
 								  cheapest_startup_outer,
 								  cheapest_total_inner,
 								  restrictlist,
@@ -1219,6 +1298,7 @@ hash_inner_and_outer(PlannerInfo *root,
 								  sjinfo,
 								  semifactors,
 								  param_source_rels,
+								  extra_lateral_rels,
 								  cheapest_startup_outer,
 								  cheapest_total_inner,
 								  restrictlist,
@@ -1256,6 +1336,7 @@ hash_inner_and_outer(PlannerInfo *root,
 									  sjinfo,
 									  semifactors,
 									  param_source_rels,
+									  extra_lateral_rels,
 									  outerpath,
 									  innerpath,
 									  restrictlist,