From cb92703384e2bb3fa0a690e5dbb95ad333c2b44c Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Tue, 8 Jun 2021 20:22:18 +0200 Subject: Adjust batch size in postgres_fdw to not use too many parameters The FE/BE protocol identifies parameters with an Int16 index, which limits the maximum number of parameters per query to 65535. With batching added to postges_fdw this limit is much easier to hit, as the whole batch is essentially a single query, making this error much easier to hit. The failures are a bit unpredictable, because it also depends on the number of columns in the query. So instead of just failing, this patch tweaks the batch_size to not exceed the maximum number of parameters. Reported-by: Hou Zhijie Reviewed-by: Bharath Rupireddy Discussion: https://postgr.es/m/OS0PR01MB571603973C0AC2874AD6BF2594299%40OS0PR01MB5716.jpnprd01.prod.outlook.com --- contrib/postgres_fdw/postgres_fdw.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'contrib/postgres_fdw/postgres_fdw.c') diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 0843ed9dba2..fafbab6b024 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -2030,7 +2030,7 @@ postgresGetForeignModifyBatchSize(ResultRelInfo *resultRelInfo) Assert(fmstate == NULL || fmstate->aux_fmstate == NULL); /* - * In EXPLAIN without ANALYZE, ri_fdwstate is NULL, so we have to lookup + * In EXPLAIN without ANALYZE, ri_FdwState is NULL, so we have to lookup * the option directly in server/table options. Otherwise just use the * value we determined earlier. */ @@ -2045,7 +2045,14 @@ postgresGetForeignModifyBatchSize(ResultRelInfo *resultRelInfo) resultRelInfo->ri_TrigDesc->trig_insert_after_row)) return 1; - /* Otherwise use the batch size specified for server/table. */ + /* + * Otherwise use the batch size specified for server/table. The number of + * parameters in a batch is limited to 65535 (uint16), so make sure we + * don't exceed this limit by using the maximum batch_size possible. + */ + if (fmstate && fmstate->p_nums > 0) + batch_size = Min(batch_size, PQ_QUERY_PARAM_MAX_LIMIT / fmstate->p_nums); + return batch_size; } -- cgit v1.2.3