Make ResourceOwners more easily extensible.

Instead of having a separate array/hash for each resource kind, use a
single array and hash to hold all kinds of resources. This makes it
possible to introduce new resource "kinds" without having to modify
the ResourceOwnerData struct. In particular, this makes it possible
for extensions to register custom resource kinds.

The old approach was to have a small array of resources of each kind,
and if it fills up, switch to a hash table. The new approach also uses
an array and a hash, but now the array and the hash are used at the
same time. The array is used to hold the recently added resources, and
when it fills up, they are moved to the hash. This keeps the access to
recent entries fast, even when there are a lot of long-held resources.

All the resource-specific ResourceOwnerEnlarge*(),
ResourceOwnerRemember*(), and ResourceOwnerForget*() functions have
been replaced with three generic functions that take resource kind as
argument. For convenience, we still define resource-specific wrapper
macros around the generic functions with the old names, but they are
now defined in the source files that use those resource kinds.

The release callback no longer needs to call ResourceOwnerForget on
the resource being released. ResourceOwnerRelease unregisters the
resource from the owner before calling the callback. That needed some
changes in bufmgr.c and some other files, where releasing the
resources previously always called ResourceOwnerForget.

Each resource kind specifies a release priority, and
ResourceOwnerReleaseAll releases the resources in priority order. To
make that possible, we have to restrict what you can do between
phases. After calling ResourceOwnerRelease(), you are no longer
allowed to remember any more resources in it or to forget any
previously remembered resources by calling ResourceOwnerForget.  There
was one case where that was done previously. At subtransaction commit,
AtEOSubXact_Inval() would handle the invalidation messages and call
RelationFlushRelation(), which temporarily increased the reference
count on the relation being flushed. We now switch to the parent
subtransaction's resource owner before calling AtEOSubXact_Inval(), so
that there is a valid ResourceOwner to temporarily hold that relcache
reference.

Other end-of-xact routines make similar calls to AtEOXact_Inval()
between release phases, but I didn't see any regression test failures
from those, so I'm not sure if they could reach a codepath that needs
remembering extra resources.

There were two exceptions to how the resource leak WARNINGs on commit
were printed previously: llvmjit silently released the context without
printing the warning, and a leaked buffer io triggered a PANIC. Now
everything prints a WARNING, including those cases.

Add tests in src/test/modules/test_resowner.

Reviewed-by: Aleksander Alekseev, Michael Paquier, Julien Rouhaud
Reviewed-by: Kyotaro Horiguchi, Hayato Kuroda, Álvaro Herrera, Zhihong Yu
Reviewed-by: Peter Eisentraut, Andres Freund
Discussion: https://www.postgresql.org/message-id/cbfabeb0-cd3c-e951-a572-19b365ed314d%40iki.fi

3 files changed, 210 insertions, 29 deletions

diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c
index 18db7e78e21..2e2e4d9f1f7 100644
--- a/src/backend/utils/cache/catcache.c
+++ b/src/backend/utils/cache/catcache.c
@@ -31,12 +31,13 @@
 #endif
 #include "storage/lmgr.h"
 #include "utils/builtins.h"
+#include "utils/catcache.h"
 #include "utils/datum.h"
 #include "utils/fmgroids.h"
 #include "utils/inval.h"
 #include "utils/memutils.h"
 #include "utils/rel.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 #include "utils/syscache.h"
 
 
@@ -94,6 +95,8 @@ static CatCTup *CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp,
 										uint32 hashValue, Index hashIndex,
 										bool negative);
 
+static void ReleaseCatCacheWithOwner(HeapTuple tuple, ResourceOwner resowner);
+static void ReleaseCatCacheListWithOwner(CatCList *list, ResourceOwner resowner);
 static void CatCacheFreeKeys(TupleDesc tupdesc, int nkeys, int *attnos,
 							 Datum *keys);
 static void CatCacheCopyKeys(TupleDesc tupdesc, int nkeys, int *attnos,
@@ -104,6 +107,56 @@ static void CatCacheCopyKeys(TupleDesc tupdesc, int nkeys, int *attnos,
  *					internal support functions
  */
 
+/* ResourceOwner callbacks to hold catcache references */
+
+static void ResOwnerReleaseCatCache(Datum res);
+static char *ResOwnerPrintCatCache(Datum res);
+static void ResOwnerReleaseCatCacheList(Datum res);
+static char *ResOwnerPrintCatCacheList(Datum res);
+
+static const ResourceOwnerDesc catcache_resowner_desc =
+{
+	/* catcache references */
+	.name = "catcache reference",
+	.release_phase = RESOURCE_RELEASE_AFTER_LOCKS,
+	.release_priority = RELEASE_PRIO_CATCACHE_REFS,
+	.ReleaseResource = ResOwnerReleaseCatCache,
+	.DebugPrint = ResOwnerPrintCatCache
+};
+
+static const ResourceOwnerDesc catlistref_resowner_desc =
+{
+	/* catcache-list pins */
+	.name = "catcache list reference",
+	.release_phase = RESOURCE_RELEASE_AFTER_LOCKS,
+	.release_priority = RELEASE_PRIO_CATCACHE_LIST_REFS,
+	.ReleaseResource = ResOwnerReleaseCatCacheList,
+	.DebugPrint = ResOwnerPrintCatCacheList
+};
+
+/* Convenience wrappers over ResourceOwnerRemember/Forget */
+static inline void
+ResourceOwnerRememberCatCacheRef(ResourceOwner owner, HeapTuple tuple)
+{
+	ResourceOwnerRemember(owner, PointerGetDatum(tuple), &catcache_resowner_desc);
+}
+static inline void
+ResourceOwnerForgetCatCacheRef(ResourceOwner owner, HeapTuple tuple)
+{
+	ResourceOwnerForget(owner, PointerGetDatum(tuple), &catcache_resowner_desc);
+}
+static inline void
+ResourceOwnerRememberCatCacheListRef(ResourceOwner owner, CatCList *list)
+{
+	ResourceOwnerRemember(owner, PointerGetDatum(list), &catlistref_resowner_desc);
+}
+static inline void
+ResourceOwnerForgetCatCacheListRef(ResourceOwner owner, CatCList *list)
+{
+	ResourceOwnerForget(owner, PointerGetDatum(list), &catlistref_resowner_desc);
+}
+
+
 /*
  * Hash and equality functions for system types that are used as cache key
  * fields.  In some cases, we just call the regular SQL-callable functions for
@@ -1268,7 +1321,7 @@ SearchCatCacheInternal(CatCache *cache,
 		 */
 		if (!ct->negative)
 		{
-			ResourceOwnerEnlargeCatCacheRefs(CurrentResourceOwner);
+			ResourceOwnerEnlarge(CurrentResourceOwner);
 			ct->refcount++;
 			ResourceOwnerRememberCatCacheRef(CurrentResourceOwner, &ct->tuple);
 
@@ -1369,7 +1422,7 @@ SearchCatCacheMiss(CatCache *cache,
 									 hashValue, hashIndex,
 									 false);
 		/* immediately set the refcount to 1 */
-		ResourceOwnerEnlargeCatCacheRefs(CurrentResourceOwner);
+		ResourceOwnerEnlarge(CurrentResourceOwner);
 		ct->refcount++;
 		ResourceOwnerRememberCatCacheRef(CurrentResourceOwner, &ct->tuple);
 		break;					/* assume only one match */
@@ -1437,6 +1490,12 @@ SearchCatCacheMiss(CatCache *cache,
 void
 ReleaseCatCache(HeapTuple tuple)
 {
+	ReleaseCatCacheWithOwner(tuple, CurrentResourceOwner);
+}
+
+static void
+ReleaseCatCacheWithOwner(HeapTuple tuple, ResourceOwner resowner)
+{
 	CatCTup    *ct = (CatCTup *) (((char *) tuple) -
 								  offsetof(CatCTup, tuple));
 
@@ -1445,7 +1504,8 @@ ReleaseCatCache(HeapTuple tuple)
 	Assert(ct->refcount > 0);
 
 	ct->refcount--;
-	ResourceOwnerForgetCatCacheRef(CurrentResourceOwner, &ct->tuple);
+	if (resowner)
+		ResourceOwnerForgetCatCacheRef(CurrentResourceOwner, &ct->tuple);
 
 	if (
 #ifndef CATCACHE_FORCE_RELEASE
@@ -1581,7 +1641,7 @@ SearchCatCacheList(CatCache *cache,
 		dlist_move_head(&cache->cc_lists, &cl->cache_elem);
 
 		/* Bump the list's refcount and return it */
-		ResourceOwnerEnlargeCatCacheListRefs(CurrentResourceOwner);
+		ResourceOwnerEnlarge(CurrentResourceOwner);
 		cl->refcount++;
 		ResourceOwnerRememberCatCacheListRef(CurrentResourceOwner, cl);
 
@@ -1693,7 +1753,7 @@ SearchCatCacheList(CatCache *cache,
 		table_close(relation, AccessShareLock);
 
 		/* Make sure the resource owner has room to remember this entry. */
-		ResourceOwnerEnlargeCatCacheListRefs(CurrentResourceOwner);
+		ResourceOwnerEnlarge(CurrentResourceOwner);
 
 		/* Now we can build the CatCList entry. */
 		oldcxt = MemoryContextSwitchTo(CacheMemoryContext);
@@ -1779,11 +1839,18 @@ SearchCatCacheList(CatCache *cache,
 void
 ReleaseCatCacheList(CatCList *list)
 {
+	ReleaseCatCacheListWithOwner(list, CurrentResourceOwner);
+}
+
+static void
+ReleaseCatCacheListWithOwner(CatCList *list, ResourceOwner resowner)
+{
 	/* Safety checks to ensure we were handed a cache entry */
 	Assert(list->cl_magic == CL_MAGIC);
 	Assert(list->refcount > 0);
 	list->refcount--;
-	ResourceOwnerForgetCatCacheListRef(CurrentResourceOwner, list);
+	if (resowner)
+		ResourceOwnerForgetCatCacheListRef(CurrentResourceOwner, list);
 
 	if (
 #ifndef CATCACHE_FORCE_RELEASE
@@ -2059,31 +2126,43 @@ PrepareToInvalidateCacheTuple(Relation relation,
 	}
 }
 
+/* ResourceOwner callbacks */
 
-/*
- * Subroutines for warning about reference leaks.  These are exported so
- * that resowner.c can call them.
- */
-void
-PrintCatCacheLeakWarning(HeapTuple tuple)
+static void
+ResOwnerReleaseCatCache(Datum res)
 {
+	ReleaseCatCacheWithOwner((HeapTuple) DatumGetPointer(res), NULL);
+}
+
+static char *
+ResOwnerPrintCatCache(Datum res)
+{
+	HeapTuple	tuple = (HeapTuple) DatumGetPointer(res);
 	CatCTup    *ct = (CatCTup *) (((char *) tuple) -
 								  offsetof(CatCTup, tuple));
 
 	/* Safety check to ensure we were handed a cache entry */
 	Assert(ct->ct_magic == CT_MAGIC);
 
-	elog(WARNING, "cache reference leak: cache %s (%d), tuple %u/%u has count %d",
-		 ct->my_cache->cc_relname, ct->my_cache->id,
-		 ItemPointerGetBlockNumber(&(tuple->t_self)),
-		 ItemPointerGetOffsetNumber(&(tuple->t_self)),
-		 ct->refcount);
+	return psprintf("cache %s (%d), tuple %u/%u has count %d",
+					ct->my_cache->cc_relname, ct->my_cache->id,
+					ItemPointerGetBlockNumber(&(tuple->t_self)),
+					ItemPointerGetOffsetNumber(&(tuple->t_self)),
+					ct->refcount);
 }
 
-void
-PrintCatCacheListLeakWarning(CatCList *list)
+static void
+ResOwnerReleaseCatCacheList(Datum res)
 {
-	elog(WARNING, "cache reference leak: cache %s (%d), list %p has count %d",
-		 list->my_cache->cc_relname, list->my_cache->id,
-		 list, list->refcount);
+	ReleaseCatCacheListWithOwner((CatCList *) DatumGetPointer(res), NULL);
+}
+
+static char *
+ResOwnerPrintCatCacheList(Datum res)
+{
+	CatCList   *list = (CatCList *) DatumGetPointer(res);
+
+	return psprintf("cache %s (%d), list %p has count %d",
+					list->my_cache->cc_relname, list->my_cache->id,
+					list, list->refcount);
 }
diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c
index 7d4168f82f5..8f95520f2fb 100644
--- a/src/backend/utils/cache/plancache.c
+++ b/src/backend/utils/cache/plancache.c
@@ -69,7 +69,7 @@
 #include "tcop/utility.h"
 #include "utils/inval.h"
 #include "utils/memutils.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 #include "utils/rls.h"
 #include "utils/snapmgr.h"
 #include "utils/syscache.h"
@@ -119,6 +119,31 @@ static void PlanCacheRelCallback(Datum arg, Oid relid);
 static void PlanCacheObjectCallback(Datum arg, int cacheid, uint32 hashvalue);
 static void PlanCacheSysCallback(Datum arg, int cacheid, uint32 hashvalue);
 
+/* ResourceOwner callbacks to track plancache references */
+static void ResOwnerReleaseCachedPlan(Datum res);
+
+static const ResourceOwnerDesc planref_resowner_desc =
+{
+	.name = "plancache reference",
+	.release_phase = RESOURCE_RELEASE_AFTER_LOCKS,
+	.release_priority = RELEASE_PRIO_PLANCACHE_REFS,
+	.ReleaseResource = ResOwnerReleaseCachedPlan,
+	.DebugPrint = NULL			/* the default message is fine */
+};
+
+/* Convenience wrappers over ResourceOwnerRemember/Forget */
+static inline void
+ResourceOwnerRememberPlanCacheRef(ResourceOwner owner, CachedPlan *plan)
+{
+	ResourceOwnerRemember(owner, PointerGetDatum(plan), &planref_resowner_desc);
+}
+static inline void
+ResourceOwnerForgetPlanCacheRef(ResourceOwner owner, CachedPlan *plan)
+{
+	ResourceOwnerForget(owner, PointerGetDatum(plan), &planref_resowner_desc);
+}
+
+
 /* GUC parameter */
 int			plan_cache_mode = PLAN_CACHE_MODE_AUTO;
 
@@ -1233,7 +1258,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams,
 
 	/* Flag the plan as in use by caller */
 	if (owner)
-		ResourceOwnerEnlargePlanCacheRefs(owner);
+		ResourceOwnerEnlarge(owner);
 	plan->refcount++;
 	if (owner)
 		ResourceOwnerRememberPlanCacheRef(owner, plan);
@@ -1396,7 +1421,7 @@ CachedPlanAllowsSimpleValidityCheck(CachedPlanSource *plansource,
 	/* Bump refcount if requested. */
 	if (owner)
 	{
-		ResourceOwnerEnlargePlanCacheRefs(owner);
+		ResourceOwnerEnlarge(owner);
 		plan->refcount++;
 		ResourceOwnerRememberPlanCacheRef(owner, plan);
 	}
@@ -1457,7 +1482,7 @@ CachedPlanIsSimplyValid(CachedPlanSource *plansource, CachedPlan *plan,
 	/* It's still good.  Bump refcount if requested. */
 	if (owner)
 	{
-		ResourceOwnerEnlargePlanCacheRefs(owner);
+		ResourceOwnerEnlarge(owner);
 		plan->refcount++;
 		ResourceOwnerRememberPlanCacheRef(owner, plan);
 	}
@@ -2203,3 +2228,20 @@ ResetPlanCache(void)
 		cexpr->is_valid = false;
 	}
 }
+
+/*
+ * Release all CachedPlans remembered by 'owner'
+ */
+void
+ReleaseAllPlanCacheRefsInOwner(ResourceOwner owner)
+{
+	ResourceOwnerReleaseAllOfKind(owner, &planref_resowner_desc);
+}
+
+/* ResourceOwner callbacks */
+
+static void
+ResOwnerReleaseCachedPlan(Datum res)
+{
+	ReleaseCachedPlan((CachedPlan *) DatumGetPointer(res), NULL);
+}
diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c
index a49ab465b36..b3faccbefe5 100644
--- a/src/backend/utils/cache/relcache.c
+++ b/src/backend/utils/cache/relcache.c
@@ -80,13 +80,14 @@
 #include "storage/smgr.h"
 #include "utils/array.h"
 #include "utils/builtins.h"
+#include "utils/catcache.h"
 #include "utils/datum.h"
 #include "utils/fmgroids.h"
 #include "utils/inval.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
 #include "utils/relmapper.h"
-#include "utils/resowner_private.h"
+#include "utils/resowner.h"
 #include "utils/snapmgr.h"
 #include "utils/syscache.h"
 
@@ -273,6 +274,7 @@ static HTAB *OpClassCache = NULL;
 
 /* non-export function prototypes */
 
+static void RelationCloseCleanup(Relation relation);
 static void RelationDestroyRelation(Relation relation, bool remember_tupdesc);
 static void RelationClearRelation(Relation relation, bool rebuild);
 
@@ -2115,6 +2117,31 @@ RelationIdGetRelation(Oid relationId)
  * ----------------------------------------------------------------
  */
 
+/* ResourceOwner callbacks to track relcache references */
+static void ResOwnerReleaseRelation(Datum res);
+static char *ResOwnerPrintRelCache(Datum res);
+
+static const ResourceOwnerDesc relref_resowner_desc =
+{
+	.name = "relcache reference",
+	.release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
+	.release_priority = RELEASE_PRIO_RELCACHE_REFS,
+	.ReleaseResource = ResOwnerReleaseRelation,
+	.DebugPrint = ResOwnerPrintRelCache
+};
+
+/* Convenience wrappers over ResourceOwnerRemember/Forget */
+static inline void
+ResourceOwnerRememberRelationRef(ResourceOwner owner, Relation rel)
+{
+	ResourceOwnerRemember(owner, PointerGetDatum(rel), &relref_resowner_desc);
+}
+static inline void
+ResourceOwnerForgetRelationRef(ResourceOwner owner, Relation rel)
+{
+	ResourceOwnerForget(owner, PointerGetDatum(rel), &relref_resowner_desc);
+}
+
 /*
  * RelationIncrementReferenceCount
  *		Increments relation reference count.
@@ -2126,7 +2153,7 @@ RelationIdGetRelation(Oid relationId)
 void
 RelationIncrementReferenceCount(Relation rel)
 {
-	ResourceOwnerEnlargeRelationRefs(CurrentResourceOwner);
+	ResourceOwnerEnlarge(CurrentResourceOwner);
 	rel->rd_refcnt += 1;
 	if (!IsBootstrapProcessingMode())
 		ResourceOwnerRememberRelationRef(CurrentResourceOwner, rel);
@@ -2162,6 +2189,12 @@ RelationClose(Relation relation)
 	/* Note: no locking manipulations needed */
 	RelationDecrementReferenceCount(relation);
 
+	RelationCloseCleanup(relation);
+}
+
+static void
+RelationCloseCleanup(Relation relation)
+{
 	/*
 	 * If the relation is no longer open in this session, we can clean up any
 	 * stale partition descriptors it has.  This is unlikely, so check to see
@@ -6813,3 +6846,30 @@ unlink_initfile(const char *initfilename, int elevel)
 							initfilename)));
 	}
 }
+
+/*
+ * ResourceOwner callbacks
+ */
+static char *
+ResOwnerPrintRelCache(Datum res)
+{
+	Relation	rel = (Relation) DatumGetPointer(res);
+
+	return psprintf("relation \"%s\"", RelationGetRelationName(rel));
+}
+
+static void
+ResOwnerReleaseRelation(Datum res)
+{
+	Relation	rel = (Relation) DatumGetPointer(res);
+
+	/*
+	 * This reference has already been removed from the resource owner, so
+	 * just decrement reference count without calling
+	 * ResourceOwnerForgetRelationRef.
+	 */
+	Assert(rel->rd_refcnt > 0);
+	rel->rd_refcnt -= 1;
+
+	RelationCloseCleanup((Relation) res);
+}