Add support for doing late row locking in FDWs.

Previously, FDWs could only do "early row locking", that is lock a row as
soon as it's fetched, even though local restriction/join conditions might
discard the row later.  This patch adds callbacks that allow FDWs to do
late locking in the same way that it's done for regular tables.

To make use of this feature, an FDW must support the "ctid" column as a
unique row identifier.  Currently, since ctid has to be of type TID,
the feature is of limited use, though in principle it could be used by
postgres_fdw.  We may eventually allow FDWs to specify another data type
for ctid, which would make it possible for more FDWs to use this feature.

This commit does not modify postgres_fdw to use late locking.  We've
tested some prototype code for that, but it's not in committable shape,
and besides it's quite unclear whether it actually makes sense to do late
locking against a remote server.  The extra round trips required are likely
to outweigh any benefit from improved concurrency.

Etsuro Fujita, reviewed by Ashutosh Bapat, and hacked up a lot by me

4 files changed, 38 insertions, 19 deletions

diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h
index 6c646091976..e60ab9fd963 100644
--- a/src/include/executor/executor.h
+++ b/src/include/executor/executor.h
@@ -196,7 +196,7 @@ extern void ExecConstraints(ResultRelInfo *resultRelInfo,
 extern void ExecWithCheckOptions(WCOKind kind, ResultRelInfo *resultRelInfo,
 					 TupleTableSlot *slot, EState *estate);
 extern LockTupleMode ExecUpdateLockMode(EState *estate, ResultRelInfo *relinfo);
-extern ExecRowMark *ExecFindRowMark(EState *estate, Index rti);
+extern ExecRowMark *ExecFindRowMark(EState *estate, Index rti, bool missing_ok);
 extern ExecAuxRowMark *ExecBuildAuxRowMark(ExecRowMark *erm, List *targetlist);
 extern TupleTableSlot *EvalPlanQual(EState *estate, EPQState *epqstate,
 			 Relation relation, Index rti, int lockmode,
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 511c96b0932..69b48b46778 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -89,6 +89,14 @@ typedef void (*EndForeignModify_function) (EState *estate,
 
 typedef int (*IsForeignRelUpdatable_function) (Relation rel);
 
+typedef RowMarkType (*GetForeignRowMarkType_function) (RangeTblEntry *rte,
+												LockClauseStrength strength);
+
+typedef HeapTuple (*RefetchForeignRow_function) (EState *estate,
+															 ExecRowMark *erm,
+															 Datum rowid,
+															 bool *updated);
+
 typedef void (*ExplainForeignScan_function) (ForeignScanState *node,
 													struct ExplainState *es);
 
@@ -151,6 +159,10 @@ typedef struct FdwRoutine
 	EndForeignModify_function EndForeignModify;
 	IsForeignRelUpdatable_function IsForeignRelUpdatable;
 
+	/* Functions for SELECT FOR UPDATE/SHARE row locking */
+	GetForeignRowMarkType_function GetForeignRowMarkType;
+	RefetchForeignRow_function RefetchForeignRow;
+
 	/* Support functions for EXPLAIN */
 	ExplainForeignScan_function ExplainForeignScan;
 	ExplainForeignModify_function ExplainForeignModify;
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 9de6d1484eb..5ad2cc23588 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -429,8 +429,11 @@ typedef struct EState
  * parent RTEs, which can be ignored at runtime).  Virtual relations such as
  * subqueries-in-FROM will have an ExecRowMark with relation == NULL.  See
  * PlanRowMark for details about most of the fields.  In addition to fields
- * directly derived from PlanRowMark, we store curCtid, which is used by the
- * WHERE CURRENT OF code.
+ * directly derived from PlanRowMark, we store an activity flag (to denote
+ * inactive children of inheritance trees), curCtid, which is used by the
+ * WHERE CURRENT OF code, and ermExtra, which is available for use by the plan
+ * node that sources the relation (e.g., for a foreign table the FDW can use
+ * ermExtra to hold information).
  *
  * EState->es_rowMarks is a list of these structs.
  */
@@ -442,8 +445,11 @@ typedef struct ExecRowMark
 	Index		prti;			/* parent range table index, if child */
 	Index		rowmarkId;		/* unique identifier for resjunk columns */
 	RowMarkType markType;		/* see enum in nodes/plannodes.h */
+	LockClauseStrength strength;	/* LockingClause's strength, or LCS_NONE */
 	LockWaitPolicy waitPolicy;	/* NOWAIT and SKIP LOCKED */
+	bool		ermActive;		/* is this mark relevant for current tuple? */
 	ItemPointerData curCtid;	/* ctid of currently locked tuple, if any */
+	void	   *ermExtra;		/* available for use by relation source node */
 } ExecRowMark;
 
 /*
@@ -1921,6 +1927,8 @@ typedef struct LockRowsState
 	PlanState	ps;				/* its first field is NodeTag */
 	List	   *lr_arowMarks;	/* List of ExecAuxRowMarks */
 	EPQState	lr_epqstate;	/* for evaluating EvalPlanQual rechecks */
+	HeapTuple  *lr_curtuples;	/* locked tuples (one entry per RT entry) */
+	int			lr_ntables;		/* length of lr_curtuples[] array */
 } LockRowsState;
 
 /* ----------------
diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index 9313292222a..1494b336c22 100644
--- a/src/include/nodes/plannodes.h
+++ b/src/include/nodes/plannodes.h
@@ -822,16 +822,16 @@ typedef struct Limit
  *
  * The first four of these values represent different lock strengths that
  * we can take on tuples according to SELECT FOR [KEY] UPDATE/SHARE requests.
- * We only support these on regular tables.  For foreign tables, any locking
- * that might be done for these requests must happen during the initial row
- * fetch; there is no mechanism for going back to lock a row later (and thus
- * no need for EvalPlanQual machinery during updates of foreign tables).
+ * We support these on regular tables, as well as on foreign tables whose FDWs
+ * report support for late locking.  For other foreign tables, any locking
+ * that might be done for such requests must happen during the initial row
+ * fetch; their FDWs provide no mechanism for going back to lock a row later.
  * This means that the semantics will be a bit different than for a local
  * table; in particular we are likely to lock more rows than would be locked
  * locally, since remote rows will be locked even if they then fail
- * locally-checked restriction or join quals.  However, the alternative of
- * doing a separate remote query to lock each selected row is extremely
- * unappealing, so let's do it like this for now.
+ * locally-checked restriction or join quals.  However, the prospect of
+ * doing a separate remote query to lock each selected row is usually pretty
+ * unappealing, so early locking remains a credible design choice for FDWs.
  *
  * When doing UPDATE, DELETE, or SELECT FOR UPDATE/SHARE, we have to uniquely
  * identify all the source rows, not only those from the target relations, so
@@ -840,12 +840,11 @@ typedef struct Limit
  * represented by ROW_MARK_REFERENCE.  Otherwise (for example for VALUES or
  * FUNCTION scans) we have to copy the whole row value.  ROW_MARK_COPY is
  * pretty inefficient, since most of the time we'll never need the data; but
- * fortunately the case is not performance-critical in practice.  Note that
- * we use ROW_MARK_COPY for non-target foreign tables, even if the FDW has a
- * concept of rowid and so could theoretically support some form of
- * ROW_MARK_REFERENCE.  Although copying the whole row value is inefficient,
- * it's probably still faster than doing a second remote fetch, so it doesn't
- * seem worth the extra complexity to permit ROW_MARK_REFERENCE.
+ * fortunately the overhead is usually not performance-critical in practice.
+ * By default we use ROW_MARK_COPY for foreign tables, but if the FDW has
+ * a concept of rowid it can request to use ROW_MARK_REFERENCE instead.
+ * (Again, this probably doesn't make sense if a physical remote fetch is
+ * needed, but for FDWs that map to local storage it might be credible.)
  */
 typedef enum RowMarkType
 {
@@ -866,7 +865,7 @@ typedef enum RowMarkType
  * When doing UPDATE, DELETE, or SELECT FOR UPDATE/SHARE, we create a separate
  * PlanRowMark node for each non-target relation in the query.  Relations that
  * are not specified as FOR UPDATE/SHARE are marked ROW_MARK_REFERENCE (if
- * regular tables) or ROW_MARK_COPY (if not).
+ * regular tables or supported foreign tables) or ROW_MARK_COPY (if not).
  *
  * Initially all PlanRowMarks have rti == prti and isParent == false.
  * When the planner discovers that a relation is the root of an inheritance
@@ -879,8 +878,8 @@ typedef enum RowMarkType
  * to use different markTypes).
  *
  * The planner also adds resjunk output columns to the plan that carry
- * information sufficient to identify the locked or fetched rows.  For
- * regular tables (markType != ROW_MARK_COPY), these columns are named
+ * information sufficient to identify the locked or fetched rows.  When
+ * markType != ROW_MARK_COPY, these columns are named
  *		tableoid%u			OID of table
  *		ctid%u				TID of row
  * The tableoid column is only present for an inheritance hierarchy.