Make detach-partition-concurrently-4 less timing sensitive

Same as 5e0b1aeb2dfe, for the companion test file.

This one seems lower probability (only two failures in a month of runs);
I was hardly able to reproduce a failure without a patch, so the fact
that I was also unable to reproduce one with it doesn't say anything.
We'll have to wait for further buildfarm results to see if we need any
further adjustments.

Discussion: https://postgr.es/m/20210524090712.GA3771394@rfd.leadboat.com

2 files changed, 68 insertions, 51 deletions

diff --git a/src/test/isolation/expected/detach-partition-concurrently-4.out b/src/test/isolation/expected/detach-partition-concurrently-4.out
index 21676753748..e5dc40d0769 100644
--- a/src/test/isolation/expected/detach-partition-concurrently-4.out
+++ b/src/test/isolation/expected/detach-partition-concurrently-4.out
@@ -1,6 +1,6 @@
 Parsed test spec with 3 sessions
 
-starting permutation: s2snitch s1b s1s s2detach s1cancel s1insert s1c
+starting permutation: s2snitch s1b s1s s2detach s1cancel s2noop s1insert s1c
 step s2snitch: insert into d4_pid select pg_backend_pid();
 step s1b: begin;
 step s1s: select * from d4_primary;
@@ -9,12 +9,13 @@ a
 1              
 2              
 step s2detach: alter table d4_primary detach partition d4_primary1 concurrently; <waiting ...>
-step s1cancel: select pg_cancel_backend(pid) from d4_pid;
-pg_cancel_backend
+step s1cancel: select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid;
+pg_cancel_backendpg_sleep       
 
-t              
+t                             
 step s2detach: <... completed>
 error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s2noop: UNLISTEN noop;
 step s1insert: insert into d4_fk values (1);
 ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
 step s1c: commit;
@@ -33,7 +34,7 @@ step s2detach: <... completed>
 error in steps s1insert s2detach: ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
 step s1c: commit;
 
-starting permutation: s2snitch s1brr s1s s2detach s1cancel s1insert s1c
+starting permutation: s2snitch s1brr s1s s2detach s1cancel s2noop s1insert s1c
 step s2snitch: insert into d4_pid select pg_backend_pid();
 step s1brr: begin isolation level repeatable read;
 step s1s: select * from d4_primary;
@@ -42,12 +43,13 @@ a
 1              
 2              
 step s2detach: alter table d4_primary detach partition d4_primary1 concurrently; <waiting ...>
-step s1cancel: select pg_cancel_backend(pid) from d4_pid;
-pg_cancel_backend
+step s1cancel: select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid;
+pg_cancel_backendpg_sleep       
 
-t              
+t                             
 step s2detach: <... completed>
 error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s2noop: UNLISTEN noop;
 step s1insert: insert into d4_fk values (1);
 ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
 step s1c: commit;
@@ -66,17 +68,18 @@ step s2detach: <... completed>
 error in steps s1insert s2detach: ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
 step s1c: commit;
 
-starting permutation: s2snitch s1b s1declare s2detach s1cancel s1fetchall s1insert s1c
+starting permutation: s2snitch s1b s1declare s2detach s1cancel s2noop s1fetchall s1insert s1c
 step s2snitch: insert into d4_pid select pg_backend_pid();
 step s1b: begin;
 step s1declare: declare f cursor for select * from d4_primary;
 step s2detach: alter table d4_primary detach partition d4_primary1 concurrently; <waiting ...>
-step s1cancel: select pg_cancel_backend(pid) from d4_pid;
-pg_cancel_backend
+step s1cancel: select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid;
+pg_cancel_backendpg_sleep       
 
-t              
+t                             
 step s2detach: <... completed>
 error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s2noop: UNLISTEN noop;
 step s1fetchall: fetch all from f;
 a              
 
@@ -101,17 +104,18 @@ step s2detach: <... completed>
 error in steps s1insert s2detach: ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
 step s1c: commit;
 
-starting permutation: s2snitch s1b s1declare s2detach s1cancel s1svpt s1insert s1rollback s1fetchall s1c
+starting permutation: s2snitch s1b s1declare s2detach s1cancel s2noop s1svpt s1insert s1rollback s1fetchall s1c
 step s2snitch: insert into d4_pid select pg_backend_pid();
 step s1b: begin;
 step s1declare: declare f cursor for select * from d4_primary;
 step s2detach: alter table d4_primary detach partition d4_primary1 concurrently; <waiting ...>
-step s1cancel: select pg_cancel_backend(pid) from d4_pid;
-pg_cancel_backend
+step s1cancel: select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid;
+pg_cancel_backendpg_sleep       
 
-t              
+t                             
 step s2detach: <... completed>
 error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s2noop: UNLISTEN noop;
 step s1svpt: savepoint f;
 step s1insert: insert into d4_fk values (1);
 ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
@@ -140,15 +144,16 @@ a
 step s1c: commit;
 step s2detach: <... completed>
 
-starting permutation: s2snitch s1b s2detach s1declare s1cancel s1fetchall s1insert s1c
+starting permutation: s2snitch s1b s2detach s1declare s1cancel s2noop s1fetchall s1insert s1c
 step s2snitch: insert into d4_pid select pg_backend_pid();
 step s1b: begin;
 step s2detach: alter table d4_primary detach partition d4_primary1 concurrently;
 step s1declare: declare f cursor for select * from d4_primary;
-step s1cancel: select pg_cancel_backend(pid) from d4_pid;
-pg_cancel_backend
+step s1cancel: select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid;
+pg_cancel_backendpg_sleep       
 
-t              
+t                             
+step s2noop: UNLISTEN noop;
 step s1fetchall: fetch all from f;
 a              
 
@@ -170,15 +175,16 @@ step s1insert: insert into d4_fk values (1);
 ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
 step s1c: commit;
 
-starting permutation: s2snitch s1b s2detach s1declare s1cancel s1svpt s1insert s1rollback s1fetchall s1c
+starting permutation: s2snitch s1b s2detach s1declare s1cancel s2noop s1svpt s1insert s1rollback s1fetchall s1c
 step s2snitch: insert into d4_pid select pg_backend_pid();
 step s1b: begin;
 step s2detach: alter table d4_primary detach partition d4_primary1 concurrently;
 step s1declare: declare f cursor for select * from d4_primary;
-step s1cancel: select pg_cancel_backend(pid) from d4_pid;
-pg_cancel_backend
+step s1cancel: select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid;
+pg_cancel_backendpg_sleep       
 
-t              
+t                             
+step s2noop: UNLISTEN noop;
 step s1svpt: savepoint f;
 step s1insert: insert into d4_fk values (1);
 ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
@@ -204,7 +210,7 @@ a
 2              
 step s1c: commit;
 
-starting permutation: s2snitch s1brr s1declare2 s1fetchone s2detach s1cancel s1updcur s1c
+starting permutation: s2snitch s1brr s1declare2 s1fetchone s2detach s1cancel s2noop s1updcur s1c
 step s2snitch: insert into d4_pid select pg_backend_pid();
 step s1brr: begin isolation level repeatable read;
 step s1declare2: declare f cursor for select * from d4_fk where a = 2;
@@ -213,12 +219,13 @@ a
 
 2              
 step s2detach: alter table d4_primary detach partition d4_primary1 concurrently; <waiting ...>
-step s1cancel: select pg_cancel_backend(pid) from d4_pid;
-pg_cancel_backend
+step s1cancel: select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid;
+pg_cancel_backendpg_sleep       
 
-t              
+t                             
 step s2detach: <... completed>
 error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s2noop: UNLISTEN noop;
 step s1updcur: update d4_fk set a = 1 where current of f;
 ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
 step s1c: commit;
@@ -265,7 +272,7 @@ ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk
 step s1c: commit;
 step s2detach: <... completed>
 
-starting permutation: s2snitch s1b s1s s2detach s3brr s3insert s3commit s1cancel s1c
+starting permutation: s2snitch s1b s1s s2detach s3brr s3insert s3commit s1cancel s2noop s1c
 step s2snitch: insert into d4_pid select pg_backend_pid();
 step s1b: begin;
 step s1s: select * from d4_primary;
@@ -278,12 +285,13 @@ step s3brr: begin isolation level repeatable read;
 step s3insert: insert into d4_fk values (1);
 ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
 step s3commit: commit;
-step s1cancel: select pg_cancel_backend(pid) from d4_pid;
-pg_cancel_backend
+step s1cancel: select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid;
+pg_cancel_backendpg_sleep       
 
-t              
+t                             
 step s2detach: <... completed>
 error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s2noop: UNLISTEN noop;
 step s1c: commit;
 
 starting permutation: s2snitch s1b s1s s2detach s3brr s3insert s3commit s1c
@@ -302,7 +310,7 @@ step s3commit: commit;
 step s1c: commit;
 step s2detach: <... completed>
 
-starting permutation: s2snitch s1brr s1s s2detach s1cancel s3vacfreeze s1s s1insert s1c
+starting permutation: s2snitch s1brr s1s s2detach s1cancel s2noop s3vacfreeze s1s s1insert s1c
 step s2snitch: insert into d4_pid select pg_backend_pid();
 step s1brr: begin isolation level repeatable read;
 step s1s: select * from d4_primary;
@@ -311,12 +319,13 @@ a
 1              
 2              
 step s2detach: alter table d4_primary detach partition d4_primary1 concurrently; <waiting ...>
-step s1cancel: select pg_cancel_backend(pid) from d4_pid;
-pg_cancel_backend
+step s1cancel: select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid;
+pg_cancel_backendpg_sleep       
 
-t              
+t                             
 step s2detach: <... completed>
 error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s2noop: UNLISTEN noop;
 step s3vacfreeze: vacuum freeze pg_catalog.pg_inherits;
 step s1s: select * from d4_primary;
 a              
@@ -327,7 +336,7 @@ step s1insert: insert into d4_fk values (1);
 ERROR:  insert or update on table "d4_fk" violates foreign key constraint "d4_fk_a_fkey"
 step s1c: commit;
 
-starting permutation: s2snitch s1b s1s s2detach s1cancel s3vacfreeze s1s s1insert s1c
+starting permutation: s2snitch s1b s1s s2detach s1cancel s2noop s3vacfreeze s1s s1insert s1c
 step s2snitch: insert into d4_pid select pg_backend_pid();
 step s1b: begin;
 step s1s: select * from d4_primary;
@@ -336,12 +345,13 @@ a
 1              
 2              
 step s2detach: alter table d4_primary detach partition d4_primary1 concurrently; <waiting ...>
-step s1cancel: select pg_cancel_backend(pid) from d4_pid;
-pg_cancel_backend
+step s1cancel: select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid;
+pg_cancel_backendpg_sleep       
 
-t              
+t                             
 step s2detach: <... completed>
 error in steps s1cancel s2detach: ERROR:  canceling statement due to user request
+step s2noop: UNLISTEN noop;
 step s3vacfreeze: vacuum freeze pg_catalog.pg_inherits;
 step s1s: select * from d4_primary;
 a              
diff --git a/src/test/isolation/specs/detach-partition-concurrently-4.spec b/src/test/isolation/specs/detach-partition-concurrently-4.spec
index 89f5f72c8cf..5e4c0018d15 100644
--- a/src/test/isolation/specs/detach-partition-concurrently-4.spec
+++ b/src/test/isolation/specs/detach-partition-concurrently-4.spec
@@ -3,6 +3,11 @@
 # (The cases where the detaching transaction is cancelled is interesting
 # because the locking situation is completely different.  I didn't verify
 # that keeping both variants adds any extra coverage.)
+#
+# Note: Always keep "s2noop" right after "s1cancel" in permutations.  This
+# reduces the probability of the timing problem that the cancel error report
+# is shown together with the next query instead of with the cancel query.
+
 setup {
 	drop table if exists d4_primary, d4_primary1, d4_fk, d4_pid;
 	create table d4_primary (a int primary key) partition by list (a);
@@ -19,7 +24,8 @@ session "s1"
 step "s1b"			{ begin; }
 step "s1brr"		{ begin isolation level repeatable read; }
 step "s1s"			{ select * from d4_primary; }
-step "s1cancel" 	{ select pg_cancel_backend(pid) from d4_pid; }
+# Sleep 0.1s after sending cancel, to give s2 time to react
+step "s1cancel" 	{ select pg_cancel_backend(pid), pg_sleep(0.1) from d4_pid; }
 step "s1insert"		{ insert into d4_fk values (1); }
 step "s1c"			{ commit; }
 step "s1declare"	{ declare f cursor for select * from d4_primary; }
@@ -33,6 +39,7 @@ step "s1rollback"	{ rollback to f; }
 session "s2"
 step "s2snitch" { insert into d4_pid select pg_backend_pid(); }
 step "s2detach" { alter table d4_primary detach partition d4_primary1 concurrently; }
+step "s2noop"	{ UNLISTEN noop; }
 
 session "s3"
 step "s3brr"		{ begin isolation level repeatable read; }
@@ -41,34 +48,34 @@ step "s3commit"		{ commit; }
 step "s3vacfreeze"	{ vacuum freeze pg_catalog.pg_inherits; }
 
 # Trying to insert into a partially detached partition is rejected
-permutation "s2snitch" "s1b"   "s1s" "s2detach" "s1cancel" "s1insert" "s1c"
+permutation "s2snitch" "s1b"   "s1s" "s2detach" "s1cancel" "s2noop" "s1insert" "s1c"
 permutation "s2snitch" "s1b"   "s1s" "s2detach"            "s1insert" "s1c"
 # ... even under REPEATABLE READ mode.
-permutation "s2snitch" "s1brr" "s1s" "s2detach" "s1cancel" "s1insert" "s1c"
+permutation "s2snitch" "s1brr" "s1s" "s2detach" "s1cancel" "s2noop" "s1insert" "s1c"
 permutation "s2snitch" "s1brr" "s1s" "s2detach"            "s1insert" "s1c"
 
 # If you read the referenced table using a cursor, you can see a row that the
 # RI query does not see.
-permutation "s2snitch" "s1b" "s1declare" "s2detach" "s1cancel" "s1fetchall" "s1insert" "s1c"
+permutation "s2snitch" "s1b" "s1declare" "s2detach" "s1cancel" "s2noop" "s1fetchall" "s1insert" "s1c"
 permutation "s2snitch" "s1b" "s1declare" "s2detach"            "s1fetchall" "s1insert" "s1c"
-permutation "s2snitch" "s1b" "s1declare" "s2detach" "s1cancel" "s1svpt" "s1insert" "s1rollback" "s1fetchall" "s1c"
+permutation "s2snitch" "s1b" "s1declare" "s2detach" "s1cancel" "s2noop" "s1svpt" "s1insert" "s1rollback" "s1fetchall" "s1c"
 permutation "s2snitch" "s1b" "s1declare" "s2detach"            "s1svpt" "s1insert" "s1rollback" "s1fetchall" "s1c"
-permutation "s2snitch" "s1b" "s2detach" "s1declare" "s1cancel" "s1fetchall" "s1insert" "s1c"
+permutation "s2snitch" "s1b" "s2detach" "s1declare" "s1cancel" "s2noop" "s1fetchall" "s1insert" "s1c"
 permutation "s2snitch" "s1b" "s2detach" "s1declare"            "s1fetchall" "s1insert" "s1c"
-permutation "s2snitch" "s1b" "s2detach" "s1declare" "s1cancel" "s1svpt" "s1insert" "s1rollback" "s1fetchall" "s1c"
+permutation "s2snitch" "s1b" "s2detach" "s1declare" "s1cancel" "s2noop" "s1svpt" "s1insert" "s1rollback" "s1fetchall" "s1c"
 permutation "s2snitch" "s1b" "s2detach" "s1declare"            "s1svpt" "s1insert" "s1rollback" "s1fetchall" "s1c"
 
 # Creating the referencing row using a cursor
-permutation "s2snitch" "s1brr" "s1declare2" "s1fetchone" "s2detach" "s1cancel" "s1updcur" "s1c"
+permutation "s2snitch" "s1brr" "s1declare2" "s1fetchone" "s2detach" "s1cancel" "s2noop" "s1updcur" "s1c"
 permutation "s2snitch" "s1brr" "s1declare2" "s1fetchone" "s2detach"            "s1updcur" "s1c"
 permutation "s2snitch" "s1brr" "s1declare2" "s1fetchone" "s1updcur" "s2detach" "s1c"
 
 # Try reading the table from an independent session.
 permutation "s2snitch" "s1b" "s1s" "s2detach" "s3insert" "s1c"
-permutation "s2snitch" "s1b" "s1s" "s2detach" "s3brr" "s3insert" "s3commit" "s1cancel" "s1c"
+permutation "s2snitch" "s1b" "s1s" "s2detach" "s3brr" "s3insert" "s3commit" "s1cancel" "s2noop" "s1c"
 permutation "s2snitch" "s1b" "s1s" "s2detach" "s3brr" "s3insert" "s3commit" "s1c"
 
 # Try one where we VACUUM FREEZE pg_inherits (to verify that xmin change is
 # handled correctly).
-permutation "s2snitch" "s1brr" "s1s" "s2detach" "s1cancel" "s3vacfreeze" "s1s" "s1insert" "s1c"
-permutation "s2snitch" "s1b" "s1s" "s2detach" "s1cancel" "s3vacfreeze" "s1s" "s1insert" "s1c"
+permutation "s2snitch" "s1brr" "s1s" "s2detach" "s1cancel" "s2noop" "s3vacfreeze" "s1s" "s1insert" "s1c"
+permutation "s2snitch" "s1b" "s1s" "s2detach" "s1cancel" "s2noop" "s3vacfreeze" "s1s" "s1insert" "s1c"