diff options
Diffstat (limited to 'src')
99 files changed, 5434 insertions, 370 deletions
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index e84c1743f4f..7ea9a77e7ea 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -2061,8 +2061,17 @@ FreeBulkInsertState(BulkInsertState bistate) * This causes rows to be frozen, which is an MVCC violation and * requires explicit options chosen by user. * + * HEAP_INSERT_IS_SPECULATIVE is used on so-called "speculative insertions", + * which can be backed out afterwards without aborting the whole transaction. + * Other sessions can wait for the speculative insertion to be confirmed, + * turning it into a regular tuple, or aborted, as if it never existed. + * Speculatively inserted tuples behave as "value locks" of short duration, + * used to implement INSERT .. ON CONFLICT. + * * Note that these options will be applied when inserting into the heap's * TOAST table, too, if the tuple requires any out-of-line data. + * FIXME: Do we mark TOAST tuples as speculative too? What about confirming + * or aborting them? * * The BulkInsertState object (if any; bistate can be NULL for default * behavior) is also just passed through to RelationGetBufferForTuple. @@ -2115,7 +2124,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, /* NO EREPORT(ERROR) from here till changes are logged */ START_CRIT_SECTION(); - RelationPutHeapTuple(relation, buffer, heaptup); + RelationPutHeapTuple(relation, buffer, heaptup, + (options & HEAP_INSERT_SPECULATIVE) != 0); if (PageIsAllVisible(BufferGetPage(buffer))) { @@ -2169,7 +2179,11 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, } xlrec.offnum = ItemPointerGetOffsetNumber(&heaptup->t_self); - xlrec.flags = all_visible_cleared ? XLOG_HEAP_ALL_VISIBLE_CLEARED : 0; + xlrec.flags = 0; + if (all_visible_cleared) + xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED; + if (options & HEAP_INSERT_SPECULATIVE) + xlrec.flags |= XLH_INSERT_IS_SPECULATIVE; Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer)); /* @@ -2179,7 +2193,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, */ if (RelationIsLogicallyLogged(relation)) { - xlrec.flags |= XLOG_HEAP_CONTAINS_NEW_TUPLE; + xlrec.flags |= XLH_INSERT_CONTAINS_NEW_TUPLE; bufflags |= REGBUF_KEEP_DATA; } @@ -2224,6 +2238,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, */ CacheInvalidateHeapTuple(relation, heaptup, NULL); + /* Note: speculative insertions are counted too, even if aborted later */ pgstat_count_heap_insert(relation, 1); /* @@ -2395,7 +2410,7 @@ heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, * RelationGetBufferForTuple has ensured that the first tuple fits. * Put that on the page, and then as many other tuples as fit. */ - RelationPutHeapTuple(relation, buffer, heaptuples[ndone]); + RelationPutHeapTuple(relation, buffer, heaptuples[ndone], false); for (nthispage = 1; ndone + nthispage < ntuples; nthispage++) { HeapTuple heaptup = heaptuples[ndone + nthispage]; @@ -2403,7 +2418,7 @@ heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, if (PageGetHeapFreeSpace(page) < MAXALIGN(heaptup->t_len) + saveFreeSpace) break; - RelationPutHeapTuple(relation, buffer, heaptup); + RelationPutHeapTuple(relation, buffer, heaptup, false); /* * We don't use heap_multi_insert for catalog tuples yet, but @@ -2463,7 +2478,7 @@ heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, /* the rest of the scratch space is used for tuple data */ tupledata = scratchptr; - xlrec->flags = all_visible_cleared ? XLOG_HEAP_ALL_VISIBLE_CLEARED : 0; + xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0; xlrec->ntuples = nthispage; /* @@ -2498,7 +2513,7 @@ heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, Assert((scratchptr - scratch) < BLCKSZ); if (need_tuple_data) - xlrec->flags |= XLOG_HEAP_CONTAINS_NEW_TUPLE; + xlrec->flags |= XLH_INSERT_CONTAINS_NEW_TUPLE; /* * Signal that this is the last xl_heap_multi_insert record @@ -2506,7 +2521,7 @@ heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, * decoding so it knows when to cleanup temporary data. */ if (ndone + nthispage == ntuples) - xlrec->flags |= XLOG_HEAP_LAST_MULTI_INSERT; + xlrec->flags |= XLH_INSERT_LAST_IN_MULTI; if (init) { @@ -2914,7 +2929,12 @@ l1: MarkBufferDirty(buffer); - /* XLOG stuff */ + /* + * XLOG stuff + * + * NB: heap_abort_speculative() uses the same xlog record and replay + * routines. + */ if (RelationNeedsWAL(relation)) { xl_heap_delete xlrec; @@ -2924,7 +2944,7 @@ l1: if (RelationIsAccessibleInLogicalDecoding(relation)) log_heap_new_cid(relation, &tp); - xlrec.flags = all_visible_cleared ? XLOG_HEAP_ALL_VISIBLE_CLEARED : 0; + xlrec.flags = all_visible_cleared ? XLH_DELETE_ALL_VISIBLE_CLEARED : 0; xlrec.infobits_set = compute_infobits(tp.t_data->t_infomask, tp.t_data->t_infomask2); xlrec.offnum = ItemPointerGetOffsetNumber(&tp.t_self); @@ -2933,9 +2953,9 @@ l1: if (old_key_tuple != NULL) { if (relation->rd_rel->relreplident == REPLICA_IDENTITY_FULL) - xlrec.flags |= XLOG_HEAP_CONTAINS_OLD_TUPLE; + xlrec.flags |= XLH_DELETE_CONTAINS_OLD_TUPLE; else - xlrec.flags |= XLOG_HEAP_CONTAINS_OLD_KEY; + xlrec.flags |= XLH_DELETE_CONTAINS_OLD_KEY; } XLogBeginInsert(); @@ -3742,7 +3762,7 @@ l2: HeapTupleClearHeapOnly(newtup); } - RelationPutHeapTuple(relation, newbuf, heaptup); /* insert new tuple */ + RelationPutHeapTuple(relation, newbuf, heaptup, false); /* insert new tuple */ if (!already_marked) { @@ -4133,14 +4153,16 @@ get_mxact_status_for_lock(LockTupleMode mode, bool is_update) * * Function result may be: * HeapTupleMayBeUpdated: lock was successfully acquired + * HeapTupleInvisible: lock failed because tuple was never visible to us * HeapTupleSelfUpdated: lock failed because tuple updated by self * HeapTupleUpdated: lock failed because tuple updated by other xact * HeapTupleWouldBlock: lock couldn't be acquired and wait_policy is skip * - * In the failure cases, the routine fills *hufd with the tuple's t_ctid, - * t_xmax (resolving a possible MultiXact, if necessary), and t_cmax - * (the last only for HeapTupleSelfUpdated, since we - * cannot obtain cmax from a combocid generated by another transaction). + * In the failure cases other than HeapTupleInvisible, the routine fills + * *hufd with the tuple's t_ctid, t_xmax (resolving a possible MultiXact, + * if necessary), and t_cmax (the last only for HeapTupleSelfUpdated, + * since we cannot obtain cmax from a combocid generated by another + * transaction). * See comments for struct HeapUpdateFailureData for additional info. * * See README.tuplock for a thorough explanation of this mechanism. @@ -4179,8 +4201,15 @@ l3: if (result == HeapTupleInvisible) { - UnlockReleaseBuffer(*buffer); - elog(ERROR, "attempted to lock invisible tuple"); + LockBuffer(*buffer, BUFFER_LOCK_UNLOCK); + + /* + * This is possible, but only when locking a tuple for ON CONFLICT + * UPDATE. We return this value here rather than throwing an error in + * order to give that case the opportunity to throw a more specific + * error. + */ + return HeapTupleInvisible; } else if (result == HeapTupleBeingUpdated) { @@ -5417,6 +5446,234 @@ heap_lock_updated_tuple(Relation rel, HeapTuple tuple, ItemPointer ctid, return HeapTupleMayBeUpdated; } +/* + * heap_finish_speculative - mark speculative insertion as successful + * + * To successfully finish a speculative insertion we have to clear speculative + * token from tuple. To do so the t_ctid field, which will contain a + * speculative token value, is modified in place to point to the tuple itself, + * which is characteristic of a newly inserted ordinary tuple. + * + * NB: It is not ok to commit without either finishing or aborting a + * speculative insertion. We could treat speculative tuples of committed + * transactions implicitly as completed, but then we would have to be prepared + * to deal with speculative tokens on committed tuples. That wouldn't be + * difficult - no-one looks at the ctid field of a tuple with invalid xmax - + * but clearing the token at completion isn't very expensive either. + * An explicit confirmation WAL record also makes logical decoding simpler. + */ +void +heap_finish_speculative(Relation relation, HeapTuple tuple) +{ + Buffer buffer; + Page page; + OffsetNumber offnum; + ItemId lp = NULL; + HeapTupleHeader htup; + + buffer = ReadBuffer(relation, ItemPointerGetBlockNumber(&(tuple->t_self))); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + page = (Page) BufferGetPage(buffer); + + offnum = ItemPointerGetOffsetNumber(&(tuple->t_self)); + if (PageGetMaxOffsetNumber(page) >= offnum) + lp = PageGetItemId(page, offnum); + + if (PageGetMaxOffsetNumber(page) < offnum || !ItemIdIsNormal(lp)) + elog(ERROR, "heap_confirm_insert: invalid lp"); + + htup = (HeapTupleHeader) PageGetItem(page, lp); + + /* SpecTokenOffsetNumber should be distinguishable from any real offset */ + StaticAssertStmt(MaxOffsetNumber < SpecTokenOffsetNumber, + "invalid speculative token constant"); + + /* NO EREPORT(ERROR) from here till changes are logged */ + START_CRIT_SECTION(); + + Assert(HeapTupleHeaderIsSpeculative(tuple->t_data)); + + MarkBufferDirty(buffer); + + /* + * Replace the speculative insertion token with a real t_ctid, + * pointing to itself like it does on regular tuples. + */ + htup->t_ctid = tuple->t_self; + + /* XLOG stuff */ + if (RelationNeedsWAL(relation)) + { + xl_heap_confirm xlrec; + XLogRecPtr recptr; + + xlrec.offnum = ItemPointerGetOffsetNumber(&tuple->t_self); + + XLogBeginInsert(); + + /* We want the same filtering on this as on a plain insert */ + XLogIncludeOrigin(); + + XLogRegisterData((char *) &xlrec, SizeOfHeapConfirm); + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + + recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_CONFIRM); + + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + + UnlockReleaseBuffer(buffer); +} + +/* + * heap_abort_speculative - kill a speculatively inserted tuple + * + * Marks a tuple that was speculatively inserted in the same command as dead, + * by setting its xmin as invalid. That makes it immediately appear as dead + * to all transactions, including our own. In particular, it makes + * HeapTupleSatisfiesDirty() regard the tuple as dead, so that another backend + * inserting a duplicate key value won't unnecessarily wait for our whole + * transaction to finish (it'll just wait for our speculative insertion to + * finish). + * + * Killing the tuple prevents "unprincipled deadlocks", which are deadlocks + * that arise due to a mutual dependency that is not user visible. By + * definition, unprincipled deadlocks cannot be prevented by the user + * reordering lock acquisition in client code, because the implementation level + * lock acquisitions are not under the user's direct control. If speculative + * inserters did not take this precaution, then under high concurrency they + * could deadlock with each other, which would not be acceptable. + * + * This is somewhat redundant with heap_delete, but we prefer to have a + * dedicated routine with stripped down requirements. + * + * This routine does not affect logical decoding as it only looks at + * confirmation records. + */ +void +heap_abort_speculative(Relation relation, HeapTuple tuple) +{ + TransactionId xid = GetCurrentTransactionId(); + ItemPointer tid = &(tuple->t_self); + ItemId lp; + HeapTupleData tp; + Page page; + BlockNumber block; + Buffer buffer; + + Assert(ItemPointerIsValid(tid)); + + block = ItemPointerGetBlockNumber(tid); + buffer = ReadBuffer(relation, block); + page = BufferGetPage(buffer); + + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + + /* + * Page can't be all visible, we just inserted into it, and are still + * running. + */ + Assert(!PageIsAllVisible(page)); + + lp = PageGetItemId(page, ItemPointerGetOffsetNumber(tid)); + Assert(ItemIdIsNormal(lp)); + + tp.t_tableOid = RelationGetRelid(relation); + tp.t_data = (HeapTupleHeader) PageGetItem(page, lp); + tp.t_len = ItemIdGetLength(lp); + tp.t_self = *tid; + + /* + * Sanity check that the tuple really is a speculatively inserted tuple, + * inserted by us. + */ + if (tp.t_data->t_choice.t_heap.t_xmin != xid) + elog(ERROR, "attempted to kill a tuple inserted by another transaction"); + if (!HeapTupleHeaderIsSpeculative(tp.t_data)) + elog(ERROR, "attempted to kill a non-speculative tuple"); + Assert(!HeapTupleHeaderIsHeapOnly(tp.t_data)); + + /* + * No need to check for serializable conflicts here. There is never a + * need for a combocid, either. No need to extract replica identity, or + * do anything special with infomask bits. + */ + + START_CRIT_SECTION(); + + /* + * The tuple will become DEAD immediately. Flag that this page + * immediately is a candidate for pruning by setting xmin to + * RecentGlobalXmin. That's not pretty, but it doesn't seem worth + * inventing a nicer API for this. + */ + Assert(TransactionIdIsValid(RecentGlobalXmin)); + PageSetPrunable(page, RecentGlobalXmin); + + /* store transaction information of xact deleting the tuple */ + tp.t_data->t_infomask &= ~(HEAP_XMAX_BITS | HEAP_MOVED); + tp.t_data->t_infomask2 &= ~HEAP_KEYS_UPDATED; + + /* + * Set the tuple header xmin to InvalidTransactionId. This makes the + * tuple immediately invisible everyone. (In particular, to any + * transactions waiting on the speculative token, woken up later.) + */ + HeapTupleHeaderSetXmin(tp.t_data, InvalidTransactionId); + + /* Clear the speculative insertion token too */ + tp.t_data->t_ctid = tp.t_self; + + MarkBufferDirty(buffer); + + /* + * XLOG stuff + * + * The WAL records generated here match heap_delete(). The same recovery + * routines are used. + */ + if (RelationNeedsWAL(relation)) + { + xl_heap_delete xlrec; + XLogRecPtr recptr; + + xlrec.flags = XLH_DELETE_IS_SUPER; + xlrec.infobits_set = compute_infobits(tp.t_data->t_infomask, + tp.t_data->t_infomask2); + xlrec.offnum = ItemPointerGetOffsetNumber(&tp.t_self); + xlrec.xmax = xid; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfHeapDelete); + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + + /* No replica identity & replication origin logged */ + + recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE); + + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + + if (HeapTupleHasExternal(&tp)) + toast_delete(relation, &tp); + + /* + * Never need to mark tuple for invalidation, since catalogs don't support + * speculative insertion + */ + + /* Now we can release the buffer */ + ReleaseBuffer(buffer); + + /* count deletion, as we counted the insertion too */ + pgstat_count_heap_delete(relation); +} /* * heap_inplace_update - update a tuple "in place" (ie, overwrite it) @@ -6732,22 +6989,22 @@ log_heap_update(Relation reln, Buffer oldbuf, /* Prepare main WAL data chain */ xlrec.flags = 0; if (all_visible_cleared) - xlrec.flags |= XLOG_HEAP_ALL_VISIBLE_CLEARED; + xlrec.flags |= XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED; if (new_all_visible_cleared) - xlrec.flags |= XLOG_HEAP_NEW_ALL_VISIBLE_CLEARED; + xlrec.flags |= XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED; if (prefixlen > 0) - xlrec.flags |= XLOG_HEAP_PREFIX_FROM_OLD; + xlrec.flags |= XLH_UPDATE_PREFIX_FROM_OLD; if (suffixlen > 0) - xlrec.flags |= XLOG_HEAP_SUFFIX_FROM_OLD; + xlrec.flags |= XLH_UPDATE_SUFFIX_FROM_OLD; if (need_tuple_data) { - xlrec.flags |= XLOG_HEAP_CONTAINS_NEW_TUPLE; + xlrec.flags |= XLH_UPDATE_CONTAINS_NEW_TUPLE; if (old_key_tuple) { if (reln->rd_rel->relreplident == REPLICA_IDENTITY_FULL) - xlrec.flags |= XLOG_HEAP_CONTAINS_OLD_TUPLE; + xlrec.flags |= XLH_UPDATE_CONTAINS_OLD_TUPLE; else - xlrec.flags |= XLOG_HEAP_CONTAINS_OLD_KEY; + xlrec.flags |= XLH_UPDATE_CONTAINS_OLD_KEY; } } @@ -7378,7 +7635,7 @@ heap_xlog_delete(XLogReaderState *record) * The visibility map may need to be fixed even if the heap page is * already up-to-date. */ - if (xlrec->flags & XLOG_HEAP_ALL_VISIBLE_CLEARED) + if (xlrec->flags & XLH_DELETE_ALL_VISIBLE_CLEARED) { Relation reln = CreateFakeRelcacheEntry(target_node); Buffer vmbuffer = InvalidBuffer; @@ -7406,13 +7663,16 @@ heap_xlog_delete(XLogReaderState *record) HeapTupleHeaderClearHotUpdated(htup); fix_infomask_from_infobits(xlrec->infobits_set, &htup->t_infomask, &htup->t_infomask2); - HeapTupleHeaderSetXmax(htup, xlrec->xmax); + if (!(xlrec->flags & XLH_DELETE_IS_SUPER)) + HeapTupleHeaderSetXmax(htup, xlrec->xmax); + else + HeapTupleHeaderSetXmin(htup, InvalidTransactionId); HeapTupleHeaderSetCmax(htup, FirstCommandId, false); /* Mark the page as a candidate for pruning */ PageSetPrunable(page, XLogRecGetXid(record)); - if (xlrec->flags & XLOG_HEAP_ALL_VISIBLE_CLEARED) + if (xlrec->flags & XLH_DELETE_ALL_VISIBLE_CLEARED) PageClearAllVisible(page); /* Make sure there is no forward chain link in t_ctid */ @@ -7453,7 +7713,7 @@ heap_xlog_insert(XLogReaderState *record) * The visibility map may need to be fixed even if the heap page is * already up-to-date. */ - if (xlrec->flags & XLOG_HEAP_ALL_VISIBLE_CLEARED) + if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) { Relation reln = CreateFakeRelcacheEntry(target_node); Buffer vmbuffer = InvalidBuffer; @@ -7516,7 +7776,7 @@ heap_xlog_insert(XLogReaderState *record) PageSetLSN(page, lsn); - if (xlrec->flags & XLOG_HEAP_ALL_VISIBLE_CLEARED) + if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) PageClearAllVisible(page); MarkBufferDirty(buffer); @@ -7573,7 +7833,7 @@ heap_xlog_multi_insert(XLogReaderState *record) * The visibility map may need to be fixed even if the heap page is * already up-to-date. */ - if (xlrec->flags & XLOG_HEAP_ALL_VISIBLE_CLEARED) + if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) { Relation reln = CreateFakeRelcacheEntry(rnode); Buffer vmbuffer = InvalidBuffer; @@ -7655,7 +7915,7 @@ heap_xlog_multi_insert(XLogReaderState *record) PageSetLSN(page, lsn); - if (xlrec->flags & XLOG_HEAP_ALL_VISIBLE_CLEARED) + if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) PageClearAllVisible(page); MarkBufferDirty(buffer); @@ -7728,7 +7988,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update) * The visibility map may need to be fixed even if the heap page is * already up-to-date. */ - if (xlrec->flags & XLOG_HEAP_ALL_VISIBLE_CLEARED) + if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) { Relation reln = CreateFakeRelcacheEntry(rnode); Buffer vmbuffer = InvalidBuffer; @@ -7783,7 +8043,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update) /* Mark the page as a candidate for pruning */ PageSetPrunable(page, XLogRecGetXid(record)); - if (xlrec->flags & XLOG_HEAP_ALL_VISIBLE_CLEARED) + if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) PageClearAllVisible(page); PageSetLSN(page, lsn); @@ -7812,7 +8072,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update) * The visibility map may need to be fixed even if the heap page is * already up-to-date. */ - if (xlrec->flags & XLOG_HEAP_NEW_ALL_VISIBLE_CLEARED) + if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) { Relation reln = CreateFakeRelcacheEntry(rnode); Buffer vmbuffer = InvalidBuffer; @@ -7840,13 +8100,13 @@ heap_xlog_update(XLogReaderState *record, bool hot_update) if (PageGetMaxOffsetNumber(page) + 1 < offnum) elog(PANIC, "heap_update_redo: invalid max offset number"); - if (xlrec->flags & XLOG_HEAP_PREFIX_FROM_OLD) + if (xlrec->flags & XLH_UPDATE_PREFIX_FROM_OLD) { Assert(newblk == oldblk); memcpy(&prefixlen, recdata, sizeof(uint16)); recdata += sizeof(uint16); } - if (xlrec->flags & XLOG_HEAP_SUFFIX_FROM_OLD) + if (xlrec->flags & XLH_UPDATE_SUFFIX_FROM_OLD) { Assert(newblk == oldblk); memcpy(&suffixlen, recdata, sizeof(uint16)); @@ -7918,7 +8178,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update) if (offnum == InvalidOffsetNumber) elog(PANIC, "heap_update_redo: failed to add tuple"); - if (xlrec->flags & XLOG_HEAP_NEW_ALL_VISIBLE_CLEARED) + if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) PageClearAllVisible(page); freespace = PageGetHeapFreeSpace(page); /* needed to update FSM below */ @@ -7952,6 +8212,42 @@ heap_xlog_update(XLogReaderState *record, bool hot_update) } static void +heap_xlog_confirm(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_heap_confirm *xlrec = (xl_heap_confirm *) XLogRecGetData(record); + Buffer buffer; + Page page; + OffsetNumber offnum; + ItemId lp = NULL; + HeapTupleHeader htup; + + if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO) + { + page = BufferGetPage(buffer); + + offnum = xlrec->offnum; + if (PageGetMaxOffsetNumber(page) >= offnum) + lp = PageGetItemId(page, offnum); + + if (PageGetMaxOffsetNumber(page) < offnum || !ItemIdIsNormal(lp)) + elog(PANIC, "heap_confirm_redo: invalid lp"); + + htup = (HeapTupleHeader) PageGetItem(page, lp); + + /* + * Confirm tuple as actually inserted + */ + ItemPointerSet(&htup->t_ctid, BufferGetBlockNumber(buffer), offnum); + + PageSetLSN(page, lsn); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); +} + +static void heap_xlog_lock(XLogReaderState *record) { XLogRecPtr lsn = record->EndRecPtr; @@ -8101,6 +8397,9 @@ heap_redo(XLogReaderState *record) case XLOG_HEAP_HOT_UPDATE: heap_xlog_update(record, true); break; + case XLOG_HEAP_CONFIRM: + heap_xlog_confirm(record); + break; case XLOG_HEAP_LOCK: heap_xlog_lock(record); break; diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c index 6d091f63af0..a9f0ca35e49 100644 --- a/src/backend/access/heap/hio.c +++ b/src/backend/access/heap/hio.c @@ -35,12 +35,17 @@ void RelationPutHeapTuple(Relation relation, Buffer buffer, - HeapTuple tuple) + HeapTuple tuple, + bool token) { Page pageHeader; OffsetNumber offnum; - ItemId itemId; - Item item; + + /* + * A tuple that's being inserted speculatively should already have its + * token set. + */ + Assert(!token || HeapTupleHeaderIsSpeculative(tuple->t_data)); /* Add the tuple to the page */ pageHeader = BufferGetPage(buffer); @@ -54,10 +59,18 @@ RelationPutHeapTuple(Relation relation, /* Update tuple->t_self to the actual position where it was stored */ ItemPointerSet(&(tuple->t_self), BufferGetBlockNumber(buffer), offnum); - /* Insert the correct position into CTID of the stored tuple, too */ - itemId = PageGetItemId(pageHeader, offnum); - item = PageGetItem(pageHeader, itemId); - ((HeapTupleHeader) item)->t_ctid = tuple->t_self; + /* + * Insert the correct position into CTID of the stored tuple, too + * (unless this is a speculative insertion, in which case the token is + * held in CTID field instead) + */ + if (!token) + { + ItemId itemId = PageGetItemId(pageHeader, offnum); + Item item = PageGetItem(pageHeader, itemId); + + ((HeapTupleHeader) item)->t_ctid = tuple->t_self; + } } /* diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c index 8464e8794f6..274155ad0c7 100644 --- a/src/backend/access/heap/tuptoaster.c +++ b/src/backend/access/heap/tuptoaster.c @@ -523,6 +523,14 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, bool toast_delold[MaxHeapAttributeNumber]; /* + * Ignore the INSERT_SPECULATIVE option. Speculative insertions/super + * deletions just normally insert/delete the toast values. It seems + * easiest to deal with that here, instead on, potentially, multiple + * callers. + */ + options &= ~HEAP_INSERT_SPECULATIVE; + + /* * We should only ever be called for tuples of plain relations or * materialized views --- recursing on a toast rel is bad news. */ diff --git a/src/backend/access/nbtree/nbtinsert.c b/src/backend/access/nbtree/nbtinsert.c index ef68a7145fc..4a60c5fa2c8 100644 --- a/src/backend/access/nbtree/nbtinsert.c +++ b/src/backend/access/nbtree/nbtinsert.c @@ -51,7 +51,8 @@ static Buffer _bt_newroot(Relation rel, Buffer lbuf, Buffer rbuf); static TransactionId _bt_check_unique(Relation rel, IndexTuple itup, Relation heapRel, Buffer buf, OffsetNumber offset, ScanKey itup_scankey, - IndexUniqueCheck checkUnique, bool *is_unique); + IndexUniqueCheck checkUnique, bool *is_unique, + uint32 *speculativeToken); static void _bt_findinsertloc(Relation rel, Buffer *bufptr, OffsetNumber *offsetptr, @@ -159,17 +160,27 @@ top: */ if (checkUnique != UNIQUE_CHECK_NO) { - TransactionId xwait; + TransactionId xwait; + uint32 speculativeToken; offset = _bt_binsrch(rel, buf, natts, itup_scankey, false); xwait = _bt_check_unique(rel, itup, heapRel, buf, offset, itup_scankey, - checkUnique, &is_unique); + checkUnique, &is_unique, &speculativeToken); if (TransactionIdIsValid(xwait)) { /* Have to wait for the other guy ... */ _bt_relbuf(rel, buf); - XactLockTableWait(xwait, rel, &itup->t_tid, XLTW_InsertIndex); + /* + * If it's a speculative insertion, wait for it to finish (ie. + * to go ahead with the insertion, or kill the tuple). Otherwise + * wait for the transaction to finish as usual. + */ + if (speculativeToken) + SpeculativeInsertionWait(xwait, speculativeToken); + else + XactLockTableWait(xwait, rel, &itup->t_tid, XLTW_InsertIndex); + /* start over... */ _bt_freestack(stack); goto top; @@ -213,7 +224,10 @@ top: * * Returns InvalidTransactionId if there is no conflict, else an xact ID * we must wait for to see if it commits a conflicting tuple. If an actual - * conflict is detected, no return --- just ereport(). + * conflict is detected, no return --- just ereport(). If an xact ID is + * returned, and the conflicting tuple still has a speculative insertion in + * progress, *speculativeToken is set to non-zero, and the caller can wait for + * the verdict on the insertion using SpeculativeInsertionWait(). * * However, if checkUnique == UNIQUE_CHECK_PARTIAL, we always return * InvalidTransactionId because we don't want to wait. In this case we @@ -223,7 +237,8 @@ top: static TransactionId _bt_check_unique(Relation rel, IndexTuple itup, Relation heapRel, Buffer buf, OffsetNumber offset, ScanKey itup_scankey, - IndexUniqueCheck checkUnique, bool *is_unique) + IndexUniqueCheck checkUnique, bool *is_unique, + uint32 *speculativeToken) { TupleDesc itupdesc = RelationGetDescr(rel); int natts = rel->rd_rel->relnatts; @@ -340,6 +355,7 @@ _bt_check_unique(Relation rel, IndexTuple itup, Relation heapRel, if (nbuf != InvalidBuffer) _bt_relbuf(rel, nbuf); /* Tell _bt_doinsert to wait... */ + *speculativeToken = SnapshotDirty.speculativeToken; return xwait; } diff --git a/src/backend/access/rmgrdesc/heapdesc.c b/src/backend/access/rmgrdesc/heapdesc.c index 4f06a2637ae..f4a1b002cf1 100644 --- a/src/backend/access/rmgrdesc/heapdesc.c +++ b/src/backend/access/rmgrdesc/heapdesc.c @@ -75,6 +75,12 @@ heap_desc(StringInfo buf, XLogReaderState *record) xlrec->new_offnum, xlrec->new_xmax); } + else if (info == XLOG_HEAP_CONFIRM) + { + xl_heap_confirm *xlrec = (xl_heap_confirm *) rec; + + appendStringInfo(buf, "off %u", xlrec->offnum); + } else if (info == XLOG_HEAP_LOCK) { xl_heap_lock *xlrec = (xl_heap_lock *) rec; @@ -177,6 +183,9 @@ heap_identify(uint8 info) case XLOG_HEAP_HOT_UPDATE | XLOG_HEAP_INIT_PAGE: id = "HOT_UPDATE+INIT"; break; + case XLOG_HEAP_CONFIRM: + id = "HEAP_CONFIRM"; + break; case XLOG_HEAP_LOCK: id = "LOCK"; break; diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index ac3b785b5a7..8c8a9eafeea 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -1665,6 +1665,10 @@ BuildIndexInfo(Relation index) /* other info */ ii->ii_Unique = indexStruct->indisunique; ii->ii_ReadyForInserts = IndexIsReady(indexStruct); + /* assume not doing speculative insertion for now */ + ii->ii_UniqueOps = NULL; + ii->ii_UniqueProcs = NULL; + ii->ii_UniqueStrats = NULL; /* initialize index-build state to default */ ii->ii_Concurrent = false; @@ -1674,6 +1678,53 @@ BuildIndexInfo(Relation index) } /* ---------------- + * BuildSpeculativeIndexInfo + * Add extra state to IndexInfo record + * + * For unique indexes, we usually don't want to add info to the IndexInfo for + * checking uniqueness, since the B-Tree AM handles that directly. However, + * in the case of speculative insertion, additional support is required. + * + * Do this processing here rather than in BuildIndexInfo() to not incur the + * overhead in the common non-speculative cases. + * ---------------- + */ +void +BuildSpeculativeIndexInfo(Relation index, IndexInfo *ii) +{ + int ncols = index->rd_rel->relnatts; + int i; + + /* + * fetch info for checking unique indexes + */ + Assert(ii->ii_Unique); + + if (index->rd_rel->relam != BTREE_AM_OID) + elog(ERROR, "unexpected non-btree speculative unique index"); + + ii->ii_UniqueOps = (Oid *) palloc(sizeof(Oid) * ncols); + ii->ii_UniqueProcs = (Oid *) palloc(sizeof(Oid) * ncols); + ii->ii_UniqueStrats = (uint16 *) palloc(sizeof(uint16) * ncols); + + /* + * We have to look up the operator's strategy number. This + * provides a cross-check that the operator does match the index. + */ + /* We need the func OIDs and strategy numbers too */ + for (i = 0; i < ncols; i++) + { + ii->ii_UniqueStrats[i] = BTEqualStrategyNumber; + ii->ii_UniqueOps[i] = + get_opfamily_member(index->rd_opfamily[i], + index->rd_opcintype[i], + index->rd_opcintype[i], + ii->ii_UniqueStrats[i]); + ii->ii_UniqueProcs[i] = get_opcode(ii->ii_UniqueOps[i]); + } +} + +/* ---------------- * FormIndexDatum * Construct values[] and isnull[] arrays for a new index tuple. * @@ -2612,7 +2663,7 @@ IndexCheckExclusion(Relation heapRelation, check_exclusion_constraint(heapRelation, indexRelation, indexInfo, &(heapTuple->t_self), values, isnull, - estate, true, false); + estate, true); } heap_endscan(scan); diff --git a/src/backend/catalog/indexing.c b/src/backend/catalog/indexing.c index fe123addac0..0231084c7c9 100644 --- a/src/backend/catalog/indexing.c +++ b/src/backend/catalog/indexing.c @@ -46,7 +46,7 @@ CatalogOpenIndexes(Relation heapRel) resultRelInfo->ri_RelationDesc = heapRel; resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */ - ExecOpenIndices(resultRelInfo); + ExecOpenIndices(resultRelInfo, false); return resultRelInfo; } diff --git a/src/backend/catalog/sql_features.txt b/src/backend/catalog/sql_features.txt index 332926424b6..cc0f8c45a6d 100644 --- a/src/backend/catalog/sql_features.txt +++ b/src/backend/catalog/sql_features.txt @@ -229,7 +229,7 @@ F311 Schema definition statement 02 CREATE TABLE for persistent base tables YES F311 Schema definition statement 03 CREATE VIEW YES F311 Schema definition statement 04 CREATE VIEW: WITH CHECK OPTION YES F311 Schema definition statement 05 GRANT statement YES -F312 MERGE statement NO +F312 MERGE statement NO Consider INSERT ... ON CONFLICT DO UPDATE F313 Enhanced MERGE statement NO F314 MERGE statement with DELETE branch NO F321 User authorization YES diff --git a/src/backend/commands/constraint.c b/src/backend/commands/constraint.c index 561d8fae574..e49affba9ee 100644 --- a/src/backend/commands/constraint.c +++ b/src/backend/commands/constraint.c @@ -172,7 +172,7 @@ unique_key_recheck(PG_FUNCTION_ARGS) */ check_exclusion_constraint(trigdata->tg_relation, indexRel, indexInfo, &(new_row->t_self), values, isnull, - estate, false, false); + estate, false); } /* diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index aa8ae4b9bcd..00a2417a099 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -2284,7 +2284,7 @@ CopyFrom(CopyState cstate) 1, /* dummy rangetable index */ 0); - ExecOpenIndices(resultRelInfo); + ExecOpenIndices(resultRelInfo, false); estate->es_result_relations = resultRelInfo; estate->es_num_result_relations = 1; @@ -2439,7 +2439,8 @@ CopyFrom(CopyState cstate) if (resultRelInfo->ri_NumIndices > 0) recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self), - estate); + estate, false, NULL, + NIL); /* AFTER ROW INSERT Triggers */ ExecARInsertTriggers(estate, resultRelInfo, tuple, @@ -2553,7 +2554,7 @@ CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid, ExecStoreTuple(bufferedTuples[i], myslot, InvalidBuffer, false); recheckIndexes = ExecInsertIndexTuples(myslot, &(bufferedTuples[i]->t_self), - estate); + estate, false, NULL, NIL); ExecARInsertTriggers(estate, resultRelInfo, bufferedTuples[i], recheckIndexes); diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index f4cc90183a4..c5452e3cb6a 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -103,7 +103,8 @@ static void ExplainIndexScanDetails(Oid indexid, ScanDirection indexorderdir, static void ExplainScanTarget(Scan *plan, ExplainState *es); static void ExplainModifyTarget(ModifyTable *plan, ExplainState *es); static void ExplainTargetRel(Plan *plan, Index rti, ExplainState *es); -static void show_modifytable_info(ModifyTableState *mtstate, ExplainState *es); +static void show_modifytable_info(ModifyTableState *mtstate, List *ancestors, + ExplainState *es); static void ExplainMemberNodes(List *plans, PlanState **planstates, List *ancestors, ExplainState *es); static void ExplainSubPlans(List *plans, List *ancestors, @@ -744,6 +745,9 @@ ExplainPreScanNode(PlanState *planstate, Bitmapset **rels_used) case T_ModifyTable: *rels_used = bms_add_member(*rels_used, ((ModifyTable *) plan)->nominalRelation); + if (((ModifyTable *) plan)->exclRelRTI) + *rels_used = bms_add_member(*rels_used, + ((ModifyTable *) plan)->exclRelRTI); break; default: break; @@ -1466,7 +1470,8 @@ ExplainNode(PlanState *planstate, List *ancestors, planstate, es); break; case T_ModifyTable: - show_modifytable_info((ModifyTableState *) planstate, es); + show_modifytable_info((ModifyTableState *) planstate, ancestors, + es); break; case T_Hash: show_hash_info((HashState *) planstate, es); @@ -2317,18 +2322,22 @@ ExplainTargetRel(Plan *plan, Index rti, ExplainState *es) /* * Show extra information for a ModifyTable node * - * We have two objectives here. First, if there's more than one target table - * or it's different from the nominal target, identify the actual target(s). - * Second, give FDWs a chance to display extra info about foreign targets. + * We have three objectives here. First, if there's more than one target + * table or it's different from the nominal target, identify the actual + * target(s). Second, give FDWs a chance to display extra info about foreign + * targets. Third, show information about ON CONFLICT. */ static void -show_modifytable_info(ModifyTableState *mtstate, ExplainState *es) +show_modifytable_info(ModifyTableState *mtstate, List *ancestors, + ExplainState *es) { ModifyTable *node = (ModifyTable *) mtstate->ps.plan; const char *operation; const char *foperation; bool labeltargets; int j; + List *idxNames = NIL; + ListCell *lst; switch (node->operation) { @@ -2414,6 +2423,55 @@ show_modifytable_info(ModifyTableState *mtstate, ExplainState *es) } } + /* Gather names of ON CONFLICT arbiter indexes */ + foreach(lst, node->arbiterIndexes) + { + char *indexname = get_rel_name(lfirst_oid(lst)); + + idxNames = lappend(idxNames, indexname); + } + + if (node->onConflictAction != ONCONFLICT_NONE) + { + ExplainProperty("Conflict Resolution", + node->onConflictAction == ONCONFLICT_NOTHING ? + "NOTHING" : "UPDATE", + false, es); + + /* + * Don't display arbiter indexes at all when DO NOTHING variant + * implicitly ignores all conflicts + */ + if (idxNames) + ExplainPropertyList("Conflict Arbiter Indexes", idxNames, es); + + /* ON CONFLICT DO UPDATE WHERE qual is specially displayed */ + if (node->onConflictWhere) + { + show_upper_qual((List *) node->onConflictWhere, "Conflict Filter", + &mtstate->ps, ancestors, es); + show_instrumentation_count("Rows Removed by Conflict Filter", 1, &mtstate->ps, es); + } + + /* EXPLAIN ANALYZE display of actual outcome for each tuple proposed */ + if (es->analyze && mtstate->ps.instrument) + { + double total; + double insert_path; + double other_path; + + InstrEndLoop(mtstate->mt_plans[0]->instrument); + + /* count the number of source rows */ + total = mtstate->mt_plans[0]->instrument->ntuples; + other_path = mtstate->ps.instrument->nfiltered2; + insert_path = total - other_path; + + ExplainPropertyFloat("Tuples Inserted", insert_path, 0, es); + ExplainPropertyFloat("Conflicting Tuples", other_path, 0, es); + } + } + if (labeltargets) ExplainCloseGroup("Target Tables", "Target Tables", false, es); } diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index 222e7fce854..b537ca5e661 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -2421,21 +2421,10 @@ ExecBRUpdateTriggers(EState *estate, EPQState *epqstate, TupleTableSlot *newSlot; int i; Bitmapset *updatedCols; - Bitmapset *keyCols; LockTupleMode lockmode; - /* - * Compute lock mode to use. If columns that are part of the key have not - * been modified, then we can use a weaker lock, allowing for better - * concurrency. - */ - updatedCols = GetUpdatedColumns(relinfo, estate); - keyCols = RelationGetIndexAttrBitmap(relinfo->ri_RelationDesc, - INDEX_ATTR_BITMAP_KEY); - if (bms_overlap(keyCols, updatedCols)) - lockmode = LockTupleExclusive; - else - lockmode = LockTupleNoKeyExclusive; + /* Determine lock mode to use */ + lockmode = ExecUpdateLockMode(estate, relinfo); Assert(HeapTupleIsValid(fdw_trigtuple) ^ ItemPointerIsValid(tupleid)); if (fdw_trigtuple == NULL) @@ -2476,6 +2465,7 @@ ExecBRUpdateTriggers(EState *estate, EPQState *epqstate, TRIGGER_EVENT_ROW | TRIGGER_EVENT_BEFORE; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; + updatedCols = GetUpdatedColumns(relinfo, estate); for (i = 0; i < trigdesc->numtriggers; i++) { Trigger *trigger = &trigdesc->triggers[i]; @@ -2783,6 +2773,9 @@ ltrmark:; */ return NULL; + case HeapTupleInvisible: + elog(ERROR, "attempted to lock invisible tuple"); + default: ReleaseBuffer(buffer); elog(ERROR, "unrecognized heap_lock_tuple status: %u", test); diff --git a/src/backend/executor/execIndexing.c b/src/backend/executor/execIndexing.c index a697682b20e..e7cf72b3875 100644 --- a/src/backend/executor/execIndexing.c +++ b/src/backend/executor/execIndexing.c @@ -50,6 +50,50 @@ * to the caller. The caller must re-check them later by calling * check_exclusion_constraint(). * + * Speculative insertion + * --------------------- + * + * Speculative insertion is a is a two-phase mechanism, used to implement + * INSERT ... ON CONFLICT DO UPDATE/NOTHING. The tuple is first inserted + * to the heap and update the indexes as usual, but if a constraint is + * violated, we can still back out the insertion without aborting the whole + * transaction. In an INSERT ... ON CONFLICT statement, if a conflict is + * detected, the inserted tuple is backed out and the ON CONFLICT action is + * executed instead. + * + * Insertion to a unique index works as usual: the index AM checks for + * duplicate keys atomically with the insertion. But instead of throwing + * an error on a conflict, the speculatively inserted heap tuple is backed + * out. + * + * Exclusion constraints are slightly more complicated. As mentioned + * earlier, there is a risk of deadlock when two backends insert the same + * key concurrently. That was not a problem for regular insertions, when + * one of the transactions has to be aborted anyway, but with a speculative + * insertion we cannot let a deadlock happen, because we only want to back + * out the speculatively inserted tuple on conflict, not abort the whole + * transaction. + * + * When a backend detects that the speculative insertion conflicts with + * another in-progress tuple, it has two options: + * + * 1. back out the speculatively inserted tuple, then wait for the other + * transaction, and retry. Or, + * 2. wait for the other transaction, with the speculatively inserted tuple + * still in place. + * + * If two backends insert at the same time, and both try to wait for each + * other, they will deadlock. So option 2 is not acceptable. Option 1 + * avoids the deadlock, but it is prone to a livelock instead. Both + * transactions will wake up immediately as the other transaction backs + * out. Then they both retry, and conflict with each other again, lather, + * rinse, repeat. + * + * To avoid the livelock, one of the backends must back out first, and then + * wait, while the other one waits without backing out. It doesn't matter + * which one backs out, so we employ an arbitrary rule that the transaction + * with the higher XID backs out. + * * * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -63,12 +107,30 @@ #include "postgres.h" #include "access/relscan.h" +#include "access/xact.h" #include "catalog/index.h" #include "executor/executor.h" #include "nodes/nodeFuncs.h" #include "storage/lmgr.h" #include "utils/tqual.h" +/* waitMode argument to check_exclusion_or_unique_constraint() */ +typedef enum +{ + CEOUC_WAIT, + CEOUC_NOWAIT, + CEOUC_LIVELOCK_PREVENTING_WAIT, +} CEOUC_WAIT_MODE; + +static bool check_exclusion_or_unique_constraint(Relation heap, Relation index, + IndexInfo *indexInfo, + ItemPointer tupleid, + Datum *values, bool *isnull, + EState *estate, bool newIndex, + CEOUC_WAIT_MODE waitMode, + bool errorOK, + ItemPointer conflictTid); + static bool index_recheck_constraint(Relation index, Oid *constr_procs, Datum *existing_values, bool *existing_isnull, Datum *new_values); @@ -84,7 +146,7 @@ static bool index_recheck_constraint(Relation index, Oid *constr_procs, * ---------------------------------------------------------------- */ void -ExecOpenIndices(ResultRelInfo *resultRelInfo) +ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative) { Relation resultRelation = resultRelInfo->ri_RelationDesc; List *indexoidlist; @@ -137,6 +199,13 @@ ExecOpenIndices(ResultRelInfo *resultRelInfo) /* extract index key information from the index's pg_index info */ ii = BuildIndexInfo(indexDesc); + /* + * If the indexes are to be used for speculative insertion, add extra + * information required by unique index entries. + */ + if (speculative && ii->ii_Unique) + BuildSpeculativeIndexInfo(indexDesc, ii); + relationDescs[i] = indexDesc; indexInfoArray[i] = ii; i++; @@ -186,7 +255,9 @@ ExecCloseIndices(ResultRelInfo *resultRelInfo) * Unique and exclusion constraints are enforced at the same * time. This returns a list of index OIDs for any unique or * exclusion constraints that are deferred and that had - * potential (unconfirmed) conflicts. + * potential (unconfirmed) conflicts. (if noDupErr == true, + * the same is done for non-deferred constraints, but report + * if conflict was speculative or deferred conflict to caller) * * CAUTION: this must not be called for a HOT update. * We can't defend against that here for lack of info. @@ -196,7 +267,10 @@ ExecCloseIndices(ResultRelInfo *resultRelInfo) List * ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, - EState *estate) + EState *estate, + bool noDupErr, + bool *specConflict, + List *arbiterIndexes) { List *result = NIL; ResultRelInfo *resultRelInfo; @@ -236,12 +310,17 @@ ExecInsertIndexTuples(TupleTableSlot *slot, IndexInfo *indexInfo; IndexUniqueCheck checkUnique; bool satisfiesConstraint; + bool arbiter; if (indexRelation == NULL) continue; indexInfo = indexInfoArray[i]; + /* Record if speculative insertion arbiter */ + arbiter = list_member_oid(arbiterIndexes, + indexRelation->rd_index->indexrelid); + /* If the index is marked as read-only, ignore it */ if (!indexInfo->ii_ReadyForInserts) continue; @@ -288,9 +367,14 @@ ExecInsertIndexTuples(TupleTableSlot *slot, * For a deferrable unique index, we tell the index AM to just detect * possible non-uniqueness, and we add the index OID to the result * list if further checking is needed. + * + * For a speculative insertion (used by INSERT ... ON CONFLICT), do + * the same as for a deferrable unique index. */ if (!indexRelation->rd_index->indisunique) checkUnique = UNIQUE_CHECK_NO; + else if (noDupErr && (arbiterIndexes == NIL || arbiter)) + checkUnique = UNIQUE_CHECK_PARTIAL; else if (indexRelation->rd_index->indimmediate) checkUnique = UNIQUE_CHECK_YES; else @@ -308,8 +392,11 @@ ExecInsertIndexTuples(TupleTableSlot *slot, * If the index has an associated exclusion constraint, check that. * This is simpler than the process for uniqueness checks since we * always insert first and then check. If the constraint is deferred, - * we check now anyway, but don't throw error on violation; instead - * we'll queue a recheck event. + * we check now anyway, but don't throw error on violation or wait for + * a conclusive outcome from a concurrent insertion; instead we'll + * queue a recheck event. Similarly, noDupErr callers (speculative + * inserters) will recheck later, and wait for a conclusive outcome + * then. * * An index for an exclusion constraint can't also be UNIQUE (not an * essential property, we just don't allow it in the grammar), so no @@ -317,13 +404,31 @@ ExecInsertIndexTuples(TupleTableSlot *slot, */ if (indexInfo->ii_ExclusionOps != NULL) { - bool errorOK = !indexRelation->rd_index->indimmediate; + bool violationOK; + bool waitMode; + + if (noDupErr) + { + violationOK = true; + waitMode = CEOUC_LIVELOCK_PREVENTING_WAIT; + } + else if (!indexRelation->rd_index->indimmediate) + { + violationOK = true; + waitMode = CEOUC_NOWAIT; + } + else + { + violationOK = false; + waitMode = CEOUC_WAIT; + } satisfiesConstraint = - check_exclusion_constraint(heapRelation, - indexRelation, indexInfo, - tupleid, values, isnull, - estate, false, errorOK); + check_exclusion_or_unique_constraint(heapRelation, + indexRelation, indexInfo, + tupleid, values, isnull, + estate, false, + waitMode, violationOK, NULL); } if ((checkUnique == UNIQUE_CHECK_PARTIAL || @@ -333,46 +438,213 @@ ExecInsertIndexTuples(TupleTableSlot *slot, /* * The tuple potentially violates the uniqueness or exclusion * constraint, so make a note of the index so that we can re-check - * it later. + * it later. Speculative inserters are told if there was a + * speculative conflict, since that always requires a restart. */ result = lappend_oid(result, RelationGetRelid(indexRelation)); + if (indexRelation->rd_index->indimmediate && specConflict) + *specConflict = true; } } return result; } +/* ---------------------------------------------------------------- + * ExecCheckIndexConstraints + * + * This routine checks if a tuple violates any unique or + * exclusion constraints. Returns true if there is no no conflict. + * Otherwise returns false, and the TID of the conflicting + * tuple is returned in *conflictTid. + * + * If 'arbiterIndexes' is given, only those indexes are checked. + * NIL means all indexes. + * + * Note that this doesn't lock the values in any way, so it's + * possible that a conflicting tuple is inserted immediately + * after this returns. But this can be used for a pre-check + * before insertion. + * ---------------------------------------------------------------- + */ +bool +ExecCheckIndexConstraints(TupleTableSlot *slot, + EState *estate, ItemPointer conflictTid, + List *arbiterIndexes) +{ + ResultRelInfo *resultRelInfo; + int i; + int numIndices; + RelationPtr relationDescs; + Relation heapRelation; + IndexInfo **indexInfoArray; + ExprContext *econtext; + Datum values[INDEX_MAX_KEYS]; + bool isnull[INDEX_MAX_KEYS]; + ItemPointerData invalidItemPtr; + bool checkedIndex = false; + + ItemPointerSetInvalid(conflictTid); + ItemPointerSetInvalid(&invalidItemPtr); + + /* + * Get information from the result relation info structure. + */ + resultRelInfo = estate->es_result_relation_info; + numIndices = resultRelInfo->ri_NumIndices; + relationDescs = resultRelInfo->ri_IndexRelationDescs; + indexInfoArray = resultRelInfo->ri_IndexRelationInfo; + heapRelation = resultRelInfo->ri_RelationDesc; + + /* + * We will use the EState's per-tuple context for evaluating predicates + * and index expressions (creating it if it's not already there). + */ + econtext = GetPerTupleExprContext(estate); + + /* Arrange for econtext's scan tuple to be the tuple under test */ + econtext->ecxt_scantuple = slot; + + /* + * For each index, form index tuple and check if it satisfies the + * constraint. + */ + for (i = 0; i < numIndices; i++) + { + Relation indexRelation = relationDescs[i]; + IndexInfo *indexInfo; + bool satisfiesConstraint; + + if (indexRelation == NULL) + continue; + + indexInfo = indexInfoArray[i]; + + if (!indexInfo->ii_Unique && !indexInfo->ii_ExclusionOps) + continue; + + /* If the index is marked as read-only, ignore it */ + if (!indexInfo->ii_ReadyForInserts) + continue; + + /* When specific arbiter indexes requested, only examine them */ + if (arbiterIndexes != NIL && + !list_member_oid(arbiterIndexes, + indexRelation->rd_index->indexrelid)) + continue; + + if (!indexRelation->rd_index->indimmediate) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ON CONFLICT does not support deferred unique constraints/exclusion constraints as arbiters"), + errtableconstraint(heapRelation, + RelationGetRelationName(indexRelation)))); + + checkedIndex = true; + + /* Check for partial index */ + if (indexInfo->ii_Predicate != NIL) + { + List *predicate; + + /* + * If predicate state not set up yet, create it (in the estate's + * per-query context) + */ + predicate = indexInfo->ii_PredicateState; + if (predicate == NIL) + { + predicate = (List *) + ExecPrepareExpr((Expr *) indexInfo->ii_Predicate, + estate); + indexInfo->ii_PredicateState = predicate; + } + + /* Skip this index-update if the predicate isn't satisfied */ + if (!ExecQual(predicate, econtext, false)) + continue; + } + + /* + * FormIndexDatum fills in its values and isnull parameters with the + * appropriate values for the column(s) of the index. + */ + FormIndexDatum(indexInfo, + slot, + estate, + values, + isnull); + + satisfiesConstraint = + check_exclusion_or_unique_constraint(heapRelation, indexRelation, + indexInfo, &invalidItemPtr, + values, isnull, estate, false, + CEOUC_WAIT, true, + conflictTid); + if (!satisfiesConstraint) + return false; + } + + if (arbiterIndexes != NIL && !checkedIndex) + elog(ERROR, "unexpected failure to find arbiter index"); + + return true; +} + /* - * Check for violation of an exclusion constraint + * Check for violation of an exclusion or unique constraint * * heap: the table containing the new tuple - * index: the index supporting the exclusion constraint + * index: the index supporting the constraint * indexInfo: info about the index, including the exclusion properties - * tupleid: heap TID of the new tuple we have just inserted + * tupleid: heap TID of the new tuple we have just inserted (invalid if we + * haven't inserted a new tuple yet) * values, isnull: the *index* column values computed for the new tuple * estate: an EState we can do evaluation in * newIndex: if true, we are trying to build a new index (this affects * only the wording of error messages) - * errorOK: if true, don't throw error for violation + * waitMode: whether to wait for concurrent inserters/deleters + * violationOK: if true, don't throw error for violation + * conflictTid: if not-NULL, the TID of the conflicting tuple is returned here * * Returns true if OK, false if actual or potential violation * - * When errorOK is true, we report violation without waiting to see if any - * concurrent transaction has committed or not; so the violation is only - * potential, and the caller must recheck sometime later. This behavior - * is convenient for deferred exclusion checks; we need not bother queuing - * a deferred event if there is definitely no conflict at insertion time. + * 'waitMode' determines what happens if a conflict is detected with a tuple + * that was inserted or deleted by a transaction that's still running. + * CEOUC_WAIT means that we wait for the transaction to commit, before + * throwing an error or returning. CEOUC_NOWAIT means that we report the + * violation immediately; so the violation is only potential, and the caller + * must recheck sometime later. This behavior is convenient for deferred + * exclusion checks; we need not bother queuing a deferred event if there is + * definitely no conflict at insertion time. + * + * CEOUC_LIVELOCK_PREVENTING_WAIT is like CEOUC_NOWAIT, but we will sometimes + * wait anyway, to prevent livelocking if two transactions try inserting at + * the same time. This is used with speculative insertions, for INSERT ON + * CONFLICT statements. (See notes in file header) * - * When errorOK is false, we'll throw error on violation, so a false result - * is impossible. + * If violationOK is true, we just report the potential or actual violation to + * the caller by returning 'false'. Otherwise we throw a descriptive error + * message here. When violationOK is false, a false result is impossible. + * + * Note: The indexam is normally responsible for checking unique constraints, + * so this normally only needs to be used for exclusion constraints. But this + * function is also called when doing a "pre-check" for conflicts on a unique + * constraint, when doing speculative insertion. Caller may use the returned + * conflict TID to take further steps. */ -bool -check_exclusion_constraint(Relation heap, Relation index, IndexInfo *indexInfo, - ItemPointer tupleid, Datum *values, bool *isnull, - EState *estate, bool newIndex, bool errorOK) +static bool +check_exclusion_or_unique_constraint(Relation heap, Relation index, + IndexInfo *indexInfo, + ItemPointer tupleid, + Datum *values, bool *isnull, + EState *estate, bool newIndex, + CEOUC_WAIT_MODE waitMode, + bool violationOK, + ItemPointer conflictTid) { - Oid *constr_procs = indexInfo->ii_ExclusionProcs; - uint16 *constr_strats = indexInfo->ii_ExclusionStrats; + Oid *constr_procs; + uint16 *constr_strats; Oid *index_collations = index->rd_indcollation; int index_natts = index->rd_index->indnatts; IndexScanDesc index_scan; @@ -386,6 +658,17 @@ check_exclusion_constraint(Relation heap, Relation index, IndexInfo *indexInfo, TupleTableSlot *existing_slot; TupleTableSlot *save_scantuple; + if (indexInfo->ii_ExclusionOps) + { + constr_procs = indexInfo->ii_ExclusionProcs; + constr_strats = indexInfo->ii_ExclusionStrats; + } + else + { + constr_procs = indexInfo->ii_UniqueProcs; + constr_strats = indexInfo->ii_UniqueStrats; + } + /* * If any of the input values are NULL, the constraint check is assumed to * pass (i.e., we assume the operators are strict). @@ -450,7 +733,8 @@ retry: /* * Ignore the entry for the tuple we're trying to check. */ - if (ItemPointerEquals(tupleid, &tup->t_self)) + if (ItemPointerIsValid(tupleid) && + ItemPointerEquals(tupleid, &tup->t_self)) { if (found_self) /* should not happen */ elog(ERROR, "found self tuple multiple times in index \"%s\"", @@ -480,39 +764,47 @@ retry: } /* - * At this point we have either a conflict or a potential conflict. If - * we're not supposed to raise error, just return the fact of the - * potential conflict without waiting to see if it's real. - */ - if (errorOK) - { - conflict = true; - break; - } - - /* + * At this point we have either a conflict or a potential conflict. + * * If an in-progress transaction is affecting the visibility of this - * tuple, we need to wait for it to complete and then recheck. For - * simplicity we do rechecking by just restarting the whole scan --- - * this case probably doesn't happen often enough to be worth trying - * harder, and anyway we don't want to hold any index internal locks - * while waiting. + * tuple, we need to wait for it to complete and then recheck (unless + * the caller requested not to). For simplicity we do rechecking by + * just restarting the whole scan --- this case probably doesn't + * happen often enough to be worth trying harder, and anyway we don't + * want to hold any index internal locks while waiting. */ xwait = TransactionIdIsValid(DirtySnapshot.xmin) ? DirtySnapshot.xmin : DirtySnapshot.xmax; - if (TransactionIdIsValid(xwait)) + if (TransactionIdIsValid(xwait) && + (waitMode == CEOUC_WAIT || + (waitMode == CEOUC_LIVELOCK_PREVENTING_WAIT && + DirtySnapshot.speculativeToken && + TransactionIdPrecedes(GetCurrentTransactionId(), xwait)))) { ctid_wait = tup->t_data->t_ctid; index_endscan(index_scan); - XactLockTableWait(xwait, heap, &ctid_wait, - XLTW_RecheckExclusionConstr); + if (DirtySnapshot.speculativeToken) + SpeculativeInsertionWait(DirtySnapshot.xmin, + DirtySnapshot.speculativeToken); + else + XactLockTableWait(xwait, heap, &ctid_wait, + XLTW_RecheckExclusionConstr); goto retry; } /* - * We have a definite conflict. Report it. + * We have a definite conflict (or a potential one, but the caller + * didn't want to wait). Return it to caller, or report it. */ + if (violationOK) + { + conflict = true; + if (conflictTid) + *conflictTid = tup->t_self; + break; + } + error_new = BuildIndexValueDescription(index, values, isnull); error_existing = BuildIndexValueDescription(index, existing_values, existing_isnull); @@ -544,10 +836,10 @@ retry: /* * Ordinarily, at this point the search should have found the originally - * inserted tuple, unless we exited the loop early because of conflict. - * However, it is possible to define exclusion constraints for which that - * wouldn't be true --- for instance, if the operator is <>. So we no - * longer complain if found_self is still false. + * inserted tuple (if any), unless we exited the loop early because of + * conflict. However, it is possible to define exclusion constraints for + * which that wouldn't be true --- for instance, if the operator is <>. + * So we no longer complain if found_self is still false. */ econtext->ecxt_scantuple = save_scantuple; @@ -558,6 +850,25 @@ retry: } /* + * Check for violation of an exclusion constraint + * + * This is a dumbed down version of check_exclusion_or_unique_constraint + * for external callers. They don't need all the special modes. + */ +void +check_exclusion_constraint(Relation heap, Relation index, + IndexInfo *indexInfo, + ItemPointer tupleid, + Datum *values, bool *isnull, + EState *estate, bool newIndex) +{ + (void) check_exclusion_or_unique_constraint(heap, index, indexInfo, tupleid, + values, isnull, + estate, newIndex, + CEOUC_WAIT, false, NULL); +} + +/* * Check existing tuple's index values to see if it really matches the * exclusion condition against the new_values. Returns true if conflict. */ diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 4272d9bc155..0dee9491788 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -1813,6 +1813,12 @@ ExecWithCheckOptions(WCOKind kind, ResultRelInfo *resultRelInfo, errmsg("new row violates row level security policy for \"%s\"", wco->relname))); break; + case WCO_RLS_CONFLICT_CHECK: + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("new row violates row level security policy (USING expression) for \"%s\"", + wco->relname))); + break; default: elog(ERROR, "unrecognized WCO kind: %u", wco->kind); break; @@ -1973,6 +1979,31 @@ ExecBuildSlotValueDescription(Oid reloid, /* + * ExecUpdateLockMode -- find the appropriate UPDATE tuple lock mode for a + * given ResultRelInfo + */ +LockTupleMode +ExecUpdateLockMode(EState *estate, ResultRelInfo *relinfo) +{ + Bitmapset *keyCols; + Bitmapset *updatedCols; + + /* + * Compute lock mode to use. If columns that are part of the key have not + * been modified, then we can use a weaker lock, allowing for better + * concurrency. + */ + updatedCols = GetUpdatedColumns(relinfo, estate); + keyCols = RelationGetIndexAttrBitmap(relinfo->ri_RelationDesc, + INDEX_ATTR_BITMAP_KEY); + + if (bms_overlap(keyCols, updatedCols)) + return LockTupleExclusive; + + return LockTupleNoKeyExclusive; +} + +/* * ExecFindRowMark -- find the ExecRowMark struct for given rangetable index */ ExecRowMark * @@ -2186,8 +2217,9 @@ EvalPlanQualFetch(EState *estate, Relation relation, int lockmode, * recycled and reused for an unrelated tuple. This implies that * the latest version of the row was deleted, so we need do * nothing. (Should be safe to examine xmin without getting - * buffer's content lock, since xmin never changes in an existing - * tuple.) + * buffer's content lock. We assume reading a TransactionId to be + * atomic, and Xmin never changes in an existing tuple, except to + * invalid or frozen, and neither of those can match priorXmax.) */ if (!TransactionIdEquals(HeapTupleHeaderGetXmin(tuple.t_data), priorXmax)) @@ -2268,11 +2300,12 @@ EvalPlanQualFetch(EState *estate, Relation relation, int lockmode, * case, so as to avoid the "Halloween problem" of * repeated update attempts. In the latter case it might * be sensible to fetch the updated tuple instead, but - * doing so would require changing heap_lock_tuple as well - * as heap_update and heap_delete to not complain about - * updating "invisible" tuples, which seems pretty scary. - * So for now, treat the tuple as deleted and do not - * process. + * doing so would require changing heap_update and + * heap_delete to not complain about updating "invisible" + * tuples, which seems pretty scary (heap_lock_tuple will + * not complain, but few callers expect HeapTupleInvisible, + * and we're not one of them). So for now, treat the tuple + * as deleted and do not process. */ ReleaseBuffer(buffer); return NULL; @@ -2287,6 +2320,9 @@ EvalPlanQualFetch(EState *estate, Relation relation, int lockmode, ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); + + /* Should not encounter speculative tuple on recheck */ + Assert(!HeapTupleHeaderIsSpeculative(tuple.t_data)); if (!ItemPointerEquals(&hufd.ctid, &tuple.t_self)) { /* it was updated, so look at the updated version */ @@ -2302,6 +2338,9 @@ EvalPlanQualFetch(EState *estate, Relation relation, int lockmode, ReleaseBuffer(buffer); return NULL; + case HeapTupleInvisible: + elog(ERROR, "attempted to lock invisible tuple"); + default: ReleaseBuffer(buffer); elog(ERROR, "unrecognized heap_lock_tuple status: %u", diff --git a/src/backend/executor/nodeLockRows.c b/src/backend/executor/nodeLockRows.c index bb6df47a95d..5ae106c06ad 100644 --- a/src/backend/executor/nodeLockRows.c +++ b/src/backend/executor/nodeLockRows.c @@ -152,10 +152,11 @@ lnext: * case, so as to avoid the "Halloween problem" of repeated * update attempts. In the latter case it might be sensible * to fetch the updated tuple instead, but doing so would - * require changing heap_lock_tuple as well as heap_update and - * heap_delete to not complain about updating "invisible" - * tuples, which seems pretty scary. So for now, treat the - * tuple as deleted and do not process. + * require changing heap_update and heap_delete to not complain + * about updating "invisible" tuples, which seems pretty scary + * (heap_lock_tuple will not complain, but few callers expect + * HeapTupleInvisible, and we're not one of them). So for now, + * treat the tuple as deleted and do not process. */ goto lnext; @@ -228,6 +229,9 @@ lnext: /* Continue loop until we have all target tuples */ break; + case HeapTupleInvisible: + elog(ERROR, "attempted to lock invisible tuple"); + default: elog(ERROR, "unrecognized heap_lock_tuple status: %u", test); diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 31666edfa8a..34435c7e50a 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -46,12 +46,22 @@ #include "miscadmin.h" #include "nodes/nodeFuncs.h" #include "storage/bufmgr.h" +#include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/memutils.h" #include "utils/rel.h" #include "utils/tqual.h" +static bool ExecOnConflictUpdate(ModifyTableState *mtstate, + ResultRelInfo *resultRelInfo, + ItemPointer conflictTid, + TupleTableSlot *planSlot, + TupleTableSlot *excludedSlot, + EState *estate, + bool canSetTag, + TupleTableSlot **returning); + /* * Verify that the tuples to be produced by INSERT or UPDATE match the * target relation's rowtype @@ -151,6 +161,51 @@ ExecProcessReturning(ProjectionInfo *projectReturning, return ExecProject(projectReturning, NULL); } +/* + * ExecCheckHeapTupleVisible -- verify heap tuple is visible + * + * It would not be consistent with guarantees of the higher isolation levels to + * proceed with avoiding insertion (taking speculative insertion's alternative + * path) on the basis of another tuple that is not visible to MVCC snapshot. + * Check for the need to raise a serialization failure, and do so as necessary. + */ +static void +ExecCheckHeapTupleVisible(EState *estate, + HeapTuple tuple, + Buffer buffer) +{ + if (!IsolationUsesXactSnapshot()) + return; + + if (!HeapTupleSatisfiesVisibility(tuple, estate->es_snapshot, buffer)) + ereport(ERROR, + (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), + errmsg("could not serialize access due to concurrent update"))); +} + +/* + * ExecCheckTIDVisible -- convenience variant of ExecCheckHeapTupleVisible() + */ +static void +ExecCheckTIDVisible(EState *estate, + ResultRelInfo *relinfo, + ItemPointer tid) +{ + Relation rel = relinfo->ri_RelationDesc; + Buffer buffer; + HeapTupleData tuple; + + /* Redundantly check isolation level */ + if (!IsolationUsesXactSnapshot()) + return; + + tuple.t_self = *tid; + if (!heap_fetch(rel, SnapshotAny, &tuple, &buffer, false, NULL)) + elog(ERROR, "failed to fetch conflicting tuple for ON CONFLICT"); + ExecCheckHeapTupleVisible(estate, &tuple, buffer); + ReleaseBuffer(buffer); +} + /* ---------------------------------------------------------------- * ExecInsert * @@ -161,8 +216,11 @@ ExecProcessReturning(ProjectionInfo *projectReturning, * ---------------------------------------------------------------- */ static TupleTableSlot * -ExecInsert(TupleTableSlot *slot, +ExecInsert(ModifyTableState *mtstate, + TupleTableSlot *slot, TupleTableSlot *planSlot, + List *arbiterIndexes, + OnConflictAction onconflict, EState *estate, bool canSetTag) { @@ -199,7 +257,15 @@ ExecInsert(TupleTableSlot *slot, if (resultRelationDesc->rd_rel->relhasoids) HeapTupleSetOid(tuple, InvalidOid); - /* BEFORE ROW INSERT Triggers */ + /* + * BEFORE ROW INSERT Triggers. + * + * Note: We fire BEFORE ROW TRIGGERS for every attempted insertion in an + * INSERT ... ON CONFLICT statement. We cannot check for constraint + * violations before firing these triggers, because they can change the + * values to insert. Also, they can run arbitrary user-defined code with + * side-effects that we can't cancel by just not inserting the tuple. + */ if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_insert_before_row) { @@ -268,21 +334,132 @@ ExecInsert(TupleTableSlot *slot, if (resultRelationDesc->rd_att->constr) ExecConstraints(resultRelInfo, slot, estate); - /* - * insert the tuple - * - * Note: heap_insert returns the tid (location) of the new tuple in - * the t_self field. - */ - newId = heap_insert(resultRelationDesc, tuple, - estate->es_output_cid, 0, NULL); + if (onconflict != ONCONFLICT_NONE && resultRelInfo->ri_NumIndices > 0) + { + /* Perform a speculative insertion. */ + uint32 specToken; + ItemPointerData conflictTid; + bool specConflict; - /* - * insert index entries for tuple - */ - if (resultRelInfo->ri_NumIndices > 0) + /* + * Do a non-conclusive check for conflicts first. + * + * We're not holding any locks yet, so this doesn't guarantee that + * the later insert won't conflict. But it avoids leaving behind + * a lot of canceled speculative insertions, if you run a lot of + * INSERT ON CONFLICT statements that do conflict. + * + * We loop back here if we find a conflict below, either during + * the pre-check, or when we re-check after inserting the tuple + * speculatively. See the executor README for a full discussion + * of speculative insertion. + */ + vlock: + specConflict = false; + if (!ExecCheckIndexConstraints(slot, estate, &conflictTid, + arbiterIndexes)) + { + /* committed conflict tuple found */ + if (onconflict == ONCONFLICT_UPDATE) + { + /* + * In case of ON CONFLICT DO UPDATE, execute the UPDATE + * part. Be prepared to retry if the UPDATE fails because + * of another concurrent UPDATE/DELETE to the conflict + * tuple. + */ + TupleTableSlot *returning = NULL; + + if (ExecOnConflictUpdate(mtstate, resultRelInfo, + &conflictTid, planSlot, slot, + estate, canSetTag, &returning)) + { + InstrCountFiltered2(&mtstate->ps, 1); + return returning; + } + else + goto vlock; + } + else + { + /* + * In case of ON CONFLICT DO NOTHING, do nothing. + * However, verify that the tuple is visible to the + * executor's MVCC snapshot at higher isolation levels. + */ + Assert(onconflict == ONCONFLICT_NOTHING); + ExecCheckTIDVisible(estate, resultRelInfo, &conflictTid); + InstrCountFiltered2(&mtstate->ps, 1); + return NULL; + } + } + + /* + * Before we start insertion proper, acquire our "speculative + * insertion lock". Others can use that to wait for us to decide + * if we're going to go ahead with the insertion, instead of + * waiting for the whole transaction to complete. + */ + specToken = SpeculativeInsertionLockAcquire(GetCurrentTransactionId()); + HeapTupleHeaderSetSpeculativeToken(tuple->t_data, specToken); + + /* insert the tuple, with the speculative token */ + newId = heap_insert(resultRelationDesc, tuple, + estate->es_output_cid, + HEAP_INSERT_SPECULATIVE, + NULL); + + /* insert index entries for tuple */ recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self), - estate); + estate, true, &specConflict, + arbiterIndexes); + + /* adjust the tuple's state accordingly */ + if (!specConflict) + heap_finish_speculative(resultRelationDesc, tuple); + else + heap_abort_speculative(resultRelationDesc, tuple); + + /* + * Wake up anyone waiting for our decision. They will re-check + * the tuple, see that it's no longer speculative, and wait on our + * XID as if this was a regularly inserted tuple all along. Or if + * we killed the tuple, they will see it's dead, and proceed as if + * the tuple never existed. + */ + SpeculativeInsertionLockRelease(GetCurrentTransactionId()); + + /* + * If there was a conflict, start from the beginning. We'll do + * the pre-check again, which will now find the conflicting tuple + * (unless it aborts before we get there). + */ + if (specConflict) + { + list_free(recheckIndexes); + goto vlock; + } + + /* Since there was no insertion conflict, we're done */ + } + else + { + /* + * insert the tuple normally. + * + * Note: heap_insert returns the tid (location) of the new tuple + * in the t_self field. + */ + newId = heap_insert(resultRelationDesc, tuple, + estate->es_output_cid, + 0, NULL); + + /* insert index entries for tuple */ + if (resultRelInfo->ri_NumIndices > 0) + recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self), + estate, false, NULL, + arbiterIndexes); + } } if (canSetTag) @@ -800,7 +977,7 @@ lreplace:; */ if (resultRelInfo->ri_NumIndices > 0 && !HeapTupleIsHeapOnly(tuple)) recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self), - estate); + estate, false, NULL, NIL); } if (canSetTag) @@ -832,6 +1009,190 @@ lreplace:; return NULL; } +/* + * ExecOnConflictUpdate --- execute UPDATE of INSERT ON CONFLICT DO UPDATE + * + * Try to lock tuple for update as part of speculative insertion. If + * a qual originating from ON CONFLICT DO UPDATE is satisfied, update + * (but still lock row, even though it may not satisfy estate's + * snapshot). + * + * Returns true if if we're done (with or without an update), or false if + * the caller must retry the INSERT from scratch. + */ +static bool +ExecOnConflictUpdate(ModifyTableState *mtstate, + ResultRelInfo *resultRelInfo, + ItemPointer conflictTid, + TupleTableSlot *planSlot, + TupleTableSlot *excludedSlot, + EState *estate, + bool canSetTag, + TupleTableSlot **returning) +{ + ExprContext *econtext = mtstate->ps.ps_ExprContext; + Relation relation = resultRelInfo->ri_RelationDesc; + List *onConflictSetWhere = resultRelInfo->ri_onConflictSetWhere; + HeapTupleData tuple; + HeapUpdateFailureData hufd; + LockTupleMode lockmode; + HTSU_Result test; + Buffer buffer; + + /* Determine lock mode to use */ + lockmode = ExecUpdateLockMode(estate, resultRelInfo); + + /* + * Lock tuple for update. Don't follow updates when tuple cannot be + * locked without doing so. A row locking conflict here means our + * previous conclusion that the tuple is conclusively committed is not + * true anymore. + */ + tuple.t_self = *conflictTid; + test = heap_lock_tuple(relation, &tuple, estate->es_output_cid, + lockmode, LockWaitBlock, false, &buffer, + &hufd); + switch (test) + { + case HeapTupleMayBeUpdated: + /* success! */ + break; + + case HeapTupleInvisible: + + /* + * This can occur when a just inserted tuple is updated again in + * the same command. E.g. because multiple rows with the same + * conflicting key values are inserted. + * + * This is somewhat similar to the ExecUpdate() + * HeapTupleSelfUpdated case. We do not want to proceed because + * it would lead to the same row being updated a second time in + * some unspecified order, and in contrast to plain UPDATEs + * there's no historical behavior to break. + * + * It is the user's responsibility to prevent this situation from + * occurring. These problems are why SQL-2003 similarly specifies + * that for SQL MERGE, an exception must be raised in the event of + * an attempt to update the same row twice. + */ + if (TransactionIdIsCurrentTransactionId(HeapTupleHeaderGetXmin(tuple.t_data))) + ereport(ERROR, + (errcode(ERRCODE_CARDINALITY_VIOLATION), + errmsg("ON CONFLICT DO UPDATE command cannot affect row a second time"), + errhint("Ensure that no rows proposed for insertion within the same command have duplicate constrained values."))); + + /* This shouldn't happen */ + elog(ERROR, "attempted to lock invisible tuple"); + + case HeapTupleSelfUpdated: + + /* + * This state should never be reached. As a dirty snapshot is used + * to find conflicting tuples, speculative insertion wouldn't have + * seen this row to conflict with. + */ + elog(ERROR, "unexpected self-updated tuple"); + + case HeapTupleUpdated: + if (IsolationUsesXactSnapshot()) + ereport(ERROR, + (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), + errmsg("could not serialize access due to concurrent update"))); + + /* + * Tell caller to try again from the very start. + * + * It does not make sense to use the usual EvalPlanQual() style + * loop here, as the new version of the row might not conflict + * anymore, or the conflicting tuple has actually been deleted. + */ + ReleaseBuffer(buffer); + return false; + + default: + elog(ERROR, "unrecognized heap_lock_tuple status: %u", test); + } + + /* + * Success, the tuple is locked. + * + * Reset per-tuple memory context to free any expression evaluation + * storage allocated in the previous cycle. + */ + ResetExprContext(econtext); + + /* + * Verify that the tuple is visible to our MVCC snapshot if the current + * isolation level mandates that. + * + * It's not sufficient to rely on the check within ExecUpdate() as e.g. + * CONFLICT ... WHERE clause may prevent us from reaching that. + * + * This means we only ever continue when a new command in the current + * transaction could see the row, even though in READ COMMITTED mode the + * tuple will not be visible according to the current statement's + * snapshot. This is in line with the way UPDATE deals with newer tuple + * versions. + */ + ExecCheckHeapTupleVisible(estate, &tuple, buffer); + + /* Store target's existing tuple in the state's dedicated slot */ + ExecStoreTuple(&tuple, mtstate->mt_existing, buffer, false); + + /* + * Make tuple and any needed join variables available to ExecQual and + * ExecProject. The EXCLUDED tuple is installed in ecxt_innertuple, while + * the target's existing tuple is installed in the scantuple. EXCLUDED has + * been made to reference INNER_VAR in setrefs.c, but there is no other + * redirection. + */ + econtext->ecxt_scantuple = mtstate->mt_existing; + econtext->ecxt_innertuple = excludedSlot; + econtext->ecxt_outertuple = NULL; + + if (!ExecQual(onConflictSetWhere, econtext, false)) + { + ReleaseBuffer(buffer); + InstrCountFiltered1(&mtstate->ps, 1); + return true; /* done with the tuple */ + } + + if (resultRelInfo->ri_WithCheckOptions != NIL) + { + /* + * Check target's existing tuple against UPDATE-applicable USING + * security barrier quals (if any), enforced here as RLS checks/WCOs. + * + * The rewriter creates UPDATE RLS checks/WCOs for UPDATE security + * quals, and stores them as WCOs of "kind" WCO_RLS_CONFLICT_CHECK, + * but that's almost the extent of its special handling for ON + * CONFLICT DO UPDATE. + * + * The rewriter will also have associated UPDATE applicable straight + * RLS checks/WCOs for the benefit of the ExecUpdate() call that + * follows. INSERTs and UPDATEs naturally have mutually exclusive WCO + * kinds, so there is no danger of spurious over-enforcement in the + * INSERT or UPDATE path. + */ + ExecWithCheckOptions(WCO_RLS_CONFLICT_CHECK, resultRelInfo, + mtstate->mt_existing, + mtstate->ps.state); + } + + /* Project the new tuple version */ + ExecProject(resultRelInfo->ri_onConflictSetProj, NULL); + + /* Execute UPDATE with projection */ + *returning = ExecUpdate(&tuple.t_data->t_ctid, NULL, + mtstate->mt_conflproj, planSlot, + &mtstate->mt_epqstate, mtstate->ps.state, + canSetTag); + + ReleaseBuffer(buffer); + return true; +} + /* * Process BEFORE EACH STATEMENT triggers @@ -843,6 +1204,9 @@ fireBSTriggers(ModifyTableState *node) { case CMD_INSERT: ExecBSInsertTriggers(node->ps.state, node->resultRelInfo); + if (node->mt_onconflict == ONCONFLICT_UPDATE) + ExecBSUpdateTriggers(node->ps.state, + node->resultRelInfo); break; case CMD_UPDATE: ExecBSUpdateTriggers(node->ps.state, node->resultRelInfo); @@ -865,6 +1229,9 @@ fireASTriggers(ModifyTableState *node) switch (node->operation) { case CMD_INSERT: + if (node->mt_onconflict == ONCONFLICT_UPDATE) + ExecASUpdateTriggers(node->ps.state, + node->resultRelInfo); ExecASInsertTriggers(node->ps.state, node->resultRelInfo); break; case CMD_UPDATE: @@ -1062,7 +1429,9 @@ ExecModifyTable(ModifyTableState *node) switch (operation) { case CMD_INSERT: - slot = ExecInsert(slot, planSlot, estate, node->canSetTag); + slot = ExecInsert(node, slot, planSlot, + node->mt_arbiterindexes, node->mt_onconflict, + estate, node->canSetTag); break; case CMD_UPDATE: slot = ExecUpdate(tupleid, oldtuple, slot, planSlot, @@ -1137,6 +1506,8 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) mtstate->resultRelInfo = estate->es_result_relations + node->resultRelIndex; mtstate->mt_arowmarks = (List **) palloc0(sizeof(List *) * nplans); mtstate->mt_nplans = nplans; + mtstate->mt_onconflict = node->onConflictAction; + mtstate->mt_arbiterindexes = node->arbiterIndexes; /* set up epqstate with dummy subplan data for the moment */ EvalPlanQualInit(&mtstate->mt_epqstate, estate, NULL, NIL, node->epqParam); @@ -1175,7 +1546,7 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) if (resultRelInfo->ri_RelationDesc->rd_rel->relhasindex && operation != CMD_DELETE && resultRelInfo->ri_IndexRelationDescs == NULL) - ExecOpenIndices(resultRelInfo); + ExecOpenIndices(resultRelInfo, mtstate->mt_onconflict != ONCONFLICT_NONE); /* Now init the plan for this result rel */ estate->es_result_relation_info = resultRelInfo; @@ -1280,6 +1651,58 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) } /* + * If needed, Initialize target list, projection and qual for ON CONFLICT + * DO UPDATE. + */ + resultRelInfo = mtstate->resultRelInfo; + if (node->onConflictAction == ONCONFLICT_UPDATE) + { + ExprContext *econtext; + ExprState *setexpr; + TupleDesc tupDesc; + + /* insert may only have one plan, inheritance is not expanded */ + Assert(nplans == 1); + + /* already exists if created by RETURNING processing above */ + if (mtstate->ps.ps_ExprContext == NULL) + ExecAssignExprContext(estate, &mtstate->ps); + + econtext = mtstate->ps.ps_ExprContext; + + /* initialize slot for the existing tuple */ + mtstate->mt_existing = ExecInitExtraTupleSlot(mtstate->ps.state); + ExecSetSlotDescriptor(mtstate->mt_existing, + resultRelInfo->ri_RelationDesc->rd_att); + + mtstate->mt_excludedtlist = node->exclRelTlist; + + /* create target slot for UPDATE SET projection */ + tupDesc = ExecTypeFromTL((List *) node->onConflictSet, + false); + mtstate->mt_conflproj = ExecInitExtraTupleSlot(mtstate->ps.state); + ExecSetSlotDescriptor(mtstate->mt_conflproj, tupDesc); + + /* build UPDATE SET expression and projection state */ + setexpr = ExecInitExpr((Expr *) node->onConflictSet, &mtstate->ps); + resultRelInfo->ri_onConflictSetProj = + ExecBuildProjectionInfo((List *) setexpr, econtext, + mtstate->mt_conflproj, + resultRelInfo->ri_RelationDesc->rd_att); + + /* build DO UPDATE WHERE clause expression */ + if (node->onConflictWhere) + { + ExprState *qualexpr; + + qualexpr = ExecInitExpr((Expr *) node->onConflictWhere, + mtstate->mt_plans[0]); + + resultRelInfo->ri_onConflictSetWhere = (List *) qualexpr; + } + } + + /* * If we have any secondary relations in an UPDATE or DELETE, they need to * be treated like non-locked relations in SELECT FOR UPDATE, ie, the * EvalPlanQual mechanism needs to be told about them. Locate the diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 1b02be287ce..a3139d3eb5d 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -81,6 +81,7 @@ _copyPlannedStmt(const PlannedStmt *from) COPY_SCALAR_FIELD(queryId); COPY_SCALAR_FIELD(hasReturning); COPY_SCALAR_FIELD(hasModifyingCTE); + COPY_SCALAR_FIELD(isUpsert); COPY_SCALAR_FIELD(canSetTag); COPY_SCALAR_FIELD(transientPlan); COPY_NODE_FIELD(planTree); @@ -185,6 +186,12 @@ _copyModifyTable(const ModifyTable *from) COPY_NODE_FIELD(fdwPrivLists); COPY_NODE_FIELD(rowMarks); COPY_SCALAR_FIELD(epqParam); + COPY_SCALAR_FIELD(onConflictAction); + COPY_NODE_FIELD(arbiterIndexes); + COPY_NODE_FIELD(onConflictSet); + COPY_NODE_FIELD(onConflictWhere); + COPY_SCALAR_FIELD(exclRelRTI); + COPY_NODE_FIELD(exclRelTlist); return newnode; } @@ -1787,6 +1794,22 @@ _copyCurrentOfExpr(const CurrentOfExpr *from) } /* + * _copyInferenceElem + */ +static InferenceElem * +_copyInferenceElem(const InferenceElem *from) +{ + InferenceElem *newnode = makeNode(InferenceElem); + + COPY_NODE_FIELD(expr); + COPY_SCALAR_FIELD(infercollid); + COPY_SCALAR_FIELD(inferopfamily); + COPY_SCALAR_FIELD(inferopcinputtype); + + return newnode; +} + +/* * _copyTargetEntry */ static TargetEntry * @@ -1852,6 +1875,26 @@ _copyFromExpr(const FromExpr *from) return newnode; } +/* + * _copyOnConflictExpr + */ +static OnConflictExpr * +_copyOnConflictExpr(const OnConflictExpr *from) +{ + OnConflictExpr *newnode = makeNode(OnConflictExpr); + + COPY_SCALAR_FIELD(action); + COPY_NODE_FIELD(arbiterElems); + COPY_NODE_FIELD(arbiterWhere); + COPY_NODE_FIELD(onConflictSet); + COPY_NODE_FIELD(onConflictWhere); + COPY_SCALAR_FIELD(constraint); + COPY_SCALAR_FIELD(exclRelIndex); + COPY_NODE_FIELD(exclRelTlist); + + return newnode; +} + /* **************************************************************** * relation.h copy functions * @@ -2135,6 +2178,33 @@ _copyWithClause(const WithClause *from) return newnode; } +static InferClause * +_copyInferClause(const InferClause *from) +{ + InferClause *newnode = makeNode(InferClause); + + COPY_NODE_FIELD(indexElems); + COPY_NODE_FIELD(whereClause); + COPY_STRING_FIELD(conname); + COPY_LOCATION_FIELD(location); + + return newnode; +} + +static OnConflictClause * +_copyOnConflictClause(const OnConflictClause *from) +{ + OnConflictClause *newnode = makeNode(OnConflictClause); + + COPY_SCALAR_FIELD(action); + COPY_NODE_FIELD(infer); + COPY_NODE_FIELD(targetList); + COPY_NODE_FIELD(whereClause); + COPY_LOCATION_FIELD(location); + + return newnode; +} + static CommonTableExpr * _copyCommonTableExpr(const CommonTableExpr *from) { @@ -2552,6 +2622,7 @@ _copyQuery(const Query *from) COPY_NODE_FIELD(jointree); COPY_NODE_FIELD(targetList); COPY_NODE_FIELD(withCheckOptions); + COPY_NODE_FIELD(onConflict); COPY_NODE_FIELD(returningList); COPY_NODE_FIELD(groupClause); COPY_NODE_FIELD(havingQual); @@ -2575,6 +2646,7 @@ _copyInsertStmt(const InsertStmt *from) COPY_NODE_FIELD(relation); COPY_NODE_FIELD(cols); COPY_NODE_FIELD(selectStmt); + COPY_NODE_FIELD(onConflictClause); COPY_NODE_FIELD(returningList); COPY_NODE_FIELD(withClause); @@ -4283,6 +4355,9 @@ copyObject(const void *from) case T_CurrentOfExpr: retval = _copyCurrentOfExpr(from); break; + case T_InferenceElem: + retval = _copyInferenceElem(from); + break; case T_TargetEntry: retval = _copyTargetEntry(from); break; @@ -4295,6 +4370,9 @@ copyObject(const void *from) case T_FromExpr: retval = _copyFromExpr(from); break; + case T_OnConflictExpr: + retval = _copyOnConflictExpr(from); + break; /* * RELATION NODES @@ -4753,6 +4831,12 @@ copyObject(const void *from) case T_WithClause: retval = _copyWithClause(from); break; + case T_InferClause: + retval = _copyInferClause(from); + break; + case T_OnConflictClause: + retval = _copyOnConflictClause(from); + break; case T_CommonTableExpr: retval = _copyCommonTableExpr(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 1b9a83b93ed..7c86e919a49 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -683,6 +683,17 @@ _equalCurrentOfExpr(const CurrentOfExpr *a, const CurrentOfExpr *b) } static bool +_equalInferenceElem(const InferenceElem *a, const InferenceElem *b) +{ + COMPARE_NODE_FIELD(expr); + COMPARE_SCALAR_FIELD(infercollid); + COMPARE_SCALAR_FIELD(inferopfamily); + COMPARE_SCALAR_FIELD(inferopcinputtype); + + return true; +} + +static bool _equalTargetEntry(const TargetEntry *a, const TargetEntry *b) { COMPARE_NODE_FIELD(expr); @@ -728,6 +739,20 @@ _equalFromExpr(const FromExpr *a, const FromExpr *b) return true; } +static bool +_equalOnConflictExpr(const OnConflictExpr *a, const OnConflictExpr *b) +{ + COMPARE_SCALAR_FIELD(action); + COMPARE_NODE_FIELD(arbiterElems); + COMPARE_NODE_FIELD(arbiterWhere); + COMPARE_NODE_FIELD(onConflictSet); + COMPARE_NODE_FIELD(onConflictWhere); + COMPARE_SCALAR_FIELD(constraint); + COMPARE_SCALAR_FIELD(exclRelIndex); + COMPARE_NODE_FIELD(exclRelTlist); + + return true; +} /* * Stuff from relation.h @@ -868,6 +893,7 @@ _equalQuery(const Query *a, const Query *b) COMPARE_NODE_FIELD(jointree); COMPARE_NODE_FIELD(targetList); COMPARE_NODE_FIELD(withCheckOptions); + COMPARE_NODE_FIELD(onConflict); COMPARE_NODE_FIELD(returningList); COMPARE_NODE_FIELD(groupClause); COMPARE_NODE_FIELD(havingQual); @@ -889,6 +915,7 @@ _equalInsertStmt(const InsertStmt *a, const InsertStmt *b) COMPARE_NODE_FIELD(relation); COMPARE_NODE_FIELD(cols); COMPARE_NODE_FIELD(selectStmt); + COMPARE_NODE_FIELD(onConflictClause); COMPARE_NODE_FIELD(returningList); COMPARE_NODE_FIELD(withClause); @@ -2434,6 +2461,29 @@ _equalWithClause(const WithClause *a, const WithClause *b) } static bool +_equalInferClause(const InferClause *a, const InferClause *b) +{ + COMPARE_NODE_FIELD(indexElems); + COMPARE_NODE_FIELD(whereClause); + COMPARE_STRING_FIELD(conname); + COMPARE_LOCATION_FIELD(location); + + return true; +} + +static bool +_equalOnConflictClause(const OnConflictClause *a, const OnConflictClause *b) +{ + COMPARE_SCALAR_FIELD(action); + COMPARE_NODE_FIELD(infer); + COMPARE_NODE_FIELD(targetList); + COMPARE_NODE_FIELD(whereClause); + COMPARE_LOCATION_FIELD(location); + + return true; +} + +static bool _equalCommonTableExpr(const CommonTableExpr *a, const CommonTableExpr *b) { COMPARE_STRING_FIELD(ctename); @@ -2712,6 +2762,9 @@ equal(const void *a, const void *b) case T_CurrentOfExpr: retval = _equalCurrentOfExpr(a, b); break; + case T_InferenceElem: + retval = _equalInferenceElem(a, b); + break; case T_TargetEntry: retval = _equalTargetEntry(a, b); break; @@ -2721,6 +2774,9 @@ equal(const void *a, const void *b) case T_FromExpr: retval = _equalFromExpr(a, b); break; + case T_OnConflictExpr: + retval = _equalOnConflictExpr(a, b); + break; case T_JoinExpr: retval = _equalJoinExpr(a, b); break; @@ -3169,6 +3225,12 @@ equal(const void *a, const void *b) case T_WithClause: retval = _equalWithClause(a, b); break; + case T_InferClause: + retval = _equalInferClause(a, b); + break; + case T_OnConflictClause: + retval = _equalOnConflictClause(a, b); + break; case T_CommonTableExpr: retval = _equalCommonTableExpr(a, b); break; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index d6f1f5bb6d7..4135f9c3cfc 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -235,6 +235,13 @@ exprType(const Node *expr) case T_CurrentOfExpr: type = BOOLOID; break; + case T_InferenceElem: + { + const InferenceElem *n = (const InferenceElem *) expr; + + type = exprType((Node *) n->expr); + } + break; case T_PlaceHolderVar: type = exprType((Node *) ((const PlaceHolderVar *) expr)->phexpr); break; @@ -894,6 +901,9 @@ exprCollation(const Node *expr) case T_CurrentOfExpr: coll = InvalidOid; /* result is always boolean */ break; + case T_InferenceElem: + coll = exprCollation((Node *) ((const InferenceElem *) expr)->expr); + break; case T_PlaceHolderVar: coll = exprCollation((Node *) ((const PlaceHolderVar *) expr)->phexpr); break; @@ -1484,6 +1494,12 @@ exprLocation(const Node *expr) case T_WithClause: loc = ((const WithClause *) expr)->location; break; + case T_InferClause: + loc = ((const InferClause *) expr)->location; + break; + case T_OnConflictClause: + loc = ((const OnConflictClause *) expr)->location; + break; case T_CommonTableExpr: loc = ((const CommonTableExpr *) expr)->location; break; @@ -1491,6 +1507,10 @@ exprLocation(const Node *expr) /* just use argument's location */ loc = exprLocation((Node *) ((const PlaceHolderVar *) expr)->phexpr); break; + case T_InferenceElem: + /* just use nested expr's location */ + loc = exprLocation((Node *) ((const InferenceElem *) expr)->expr); + break; default: /* for any other node type it's just unknown... */ loc = -1; @@ -1890,6 +1910,20 @@ expression_tree_walker(Node *node, return true; } break; + case T_OnConflictExpr: + { + OnConflictExpr *onconflict = (OnConflictExpr *) node; + + if (walker((Node *) onconflict->arbiterElems, context)) + return true; + if (walker(onconflict->arbiterWhere, context)) + return true; + if (walker(onconflict->onConflictSet, context)) + return true; + if (walker(onconflict->onConflictWhere, context)) + return true; + } + break; case T_JoinExpr: { JoinExpr *join = (JoinExpr *) node; @@ -1920,6 +1954,8 @@ expression_tree_walker(Node *node, break; case T_PlaceHolderVar: return walker(((PlaceHolderVar *) node)->phexpr, context); + case T_InferenceElem: + return walker(((InferenceElem *) node)->expr, context); case T_AppendRelInfo: { AppendRelInfo *appinfo = (AppendRelInfo *) node; @@ -1968,6 +2004,8 @@ query_tree_walker(Query *query, return true; if (walker((Node *) query->withCheckOptions, context)) return true; + if (walker((Node *) query->onConflict, context)) + return true; if (walker((Node *) query->returningList, context)) return true; if (walker((Node *) query->jointree, context)) @@ -2594,6 +2632,20 @@ expression_tree_mutator(Node *node, return (Node *) newnode; } break; + case T_OnConflictExpr: + { + OnConflictExpr *oc = (OnConflictExpr *) node; + OnConflictExpr *newnode; + + FLATCOPY(newnode, oc, OnConflictExpr); + MUTATE(newnode->arbiterElems, oc->arbiterElems, List *); + MUTATE(newnode->arbiterWhere, oc->arbiterWhere, Node *); + MUTATE(newnode->onConflictSet, oc->onConflictSet, List *); + MUTATE(newnode->onConflictWhere, oc->onConflictWhere, Node *); + + return (Node *) newnode; + } + break; case T_JoinExpr: { JoinExpr *join = (JoinExpr *) node; @@ -2630,6 +2682,16 @@ expression_tree_mutator(Node *node, return (Node *) newnode; } break; + case T_InferenceElem: + { + InferenceElem *inferenceelemdexpr = (InferenceElem *) node; + InferenceElem *newnode; + + FLATCOPY(newnode, inferenceelemdexpr, InferenceElem); + MUTATE(newnode->expr, newnode->expr, Node *); + return (Node *) newnode; + } + break; case T_AppendRelInfo: { AppendRelInfo *appinfo = (AppendRelInfo *) node; @@ -2709,6 +2771,7 @@ query_tree_mutator(Query *query, MUTATE(query->targetList, query->targetList, List *); MUTATE(query->withCheckOptions, query->withCheckOptions, List *); + MUTATE(query->onConflict, query->onConflict, OnConflictExpr *); MUTATE(query->returningList, query->returningList, List *); MUTATE(query->jointree, query->jointree, FromExpr *); MUTATE(query->setOperations, query->setOperations, Node *); @@ -2978,6 +3041,8 @@ raw_expression_tree_walker(Node *node, return true; if (walker(stmt->selectStmt, context)) return true; + if (walker(stmt->onConflictClause, context)) + return true; if (walker(stmt->returningList, context)) return true; if (walker(stmt->withClause, context)) @@ -3217,6 +3282,28 @@ raw_expression_tree_walker(Node *node, break; case T_WithClause: return walker(((WithClause *) node)->ctes, context); + case T_InferClause: + { + InferClause *stmt = (InferClause *) node; + + if (walker(stmt->indexElems, context)) + return true; + if (walker(stmt->whereClause, context)) + return true; + } + break; + case T_OnConflictClause: + { + OnConflictClause *stmt = (OnConflictClause *) node; + + if (walker(stmt->infer, context)) + return true; + if (walker(stmt->targetList, context)) + return true; + if (walker(stmt->whereClause, context)) + return true; + } + break; case T_CommonTableExpr: return walker(((CommonTableExpr *) node)->ctequery, context); default: diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d5ddd0b3592..bc891d391f5 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -243,6 +243,7 @@ _outPlannedStmt(StringInfo str, const PlannedStmt *node) WRITE_UINT_FIELD(queryId); WRITE_BOOL_FIELD(hasReturning); WRITE_BOOL_FIELD(hasModifyingCTE); + WRITE_BOOL_FIELD(isUpsert); WRITE_BOOL_FIELD(canSetTag); WRITE_BOOL_FIELD(transientPlan); WRITE_NODE_FIELD(planTree); @@ -337,6 +338,12 @@ _outModifyTable(StringInfo str, const ModifyTable *node) WRITE_NODE_FIELD(fdwPrivLists); WRITE_NODE_FIELD(rowMarks); WRITE_INT_FIELD(epqParam); + WRITE_ENUM_FIELD(onConflictAction, OnConflictAction); + WRITE_NODE_FIELD(arbiterIndexes); + WRITE_NODE_FIELD(onConflictSet); + WRITE_NODE_FIELD(onConflictWhere); + WRITE_INT_FIELD(exclRelRTI); + WRITE_NODE_FIELD(exclRelTlist); } static void @@ -1437,6 +1444,17 @@ _outCurrentOfExpr(StringInfo str, const CurrentOfExpr *node) } static void +_outInferenceElem(StringInfo str, const InferenceElem *node) +{ + WRITE_NODE_TYPE("INFERENCEELEM"); + + WRITE_NODE_FIELD(expr); + WRITE_OID_FIELD(infercollid); + WRITE_OID_FIELD(inferopfamily); + WRITE_OID_FIELD(inferopcinputtype); +} + +static void _outTargetEntry(StringInfo str, const TargetEntry *node) { WRITE_NODE_TYPE("TARGETENTRY"); @@ -1482,6 +1500,21 @@ _outFromExpr(StringInfo str, const FromExpr *node) WRITE_NODE_FIELD(quals); } +static void +_outOnConflictExpr(StringInfo str, const OnConflictExpr *node) +{ + WRITE_NODE_TYPE("ONCONFLICTEXPR"); + + WRITE_ENUM_FIELD(action, OnConflictAction); + WRITE_NODE_FIELD(arbiterElems); + WRITE_NODE_FIELD(arbiterWhere); + WRITE_NODE_FIELD(onConflictSet); + WRITE_NODE_FIELD(onConflictWhere); + WRITE_OID_FIELD(constraint); + WRITE_INT_FIELD(exclRelIndex); + WRITE_NODE_FIELD(exclRelTlist); +} + /***************************************************************************** * * Stuff from relation.h. @@ -2319,6 +2352,7 @@ _outQuery(StringInfo str, const Query *node) WRITE_NODE_FIELD(jointree); WRITE_NODE_FIELD(targetList); WRITE_NODE_FIELD(withCheckOptions); + WRITE_NODE_FIELD(onConflict); WRITE_NODE_FIELD(returningList); WRITE_NODE_FIELD(groupClause); WRITE_NODE_FIELD(havingQual); @@ -3112,6 +3146,9 @@ _outNode(StringInfo str, const void *obj) case T_CurrentOfExpr: _outCurrentOfExpr(str, obj); break; + case T_InferenceElem: + _outInferenceElem(str, obj); + break; case T_TargetEntry: _outTargetEntry(str, obj); break; @@ -3124,7 +3161,9 @@ _outNode(StringInfo str, const void *obj) case T_FromExpr: _outFromExpr(str, obj); break; - + case T_OnConflictExpr: + _outOnConflictExpr(str, obj); + break; case T_Path: _outPath(str, obj); break; diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index d1ced0cc4b3..8136306e1e5 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -214,6 +214,7 @@ _readQuery(void) READ_NODE_FIELD(jointree); READ_NODE_FIELD(targetList); READ_NODE_FIELD(withCheckOptions); + READ_NODE_FIELD(onConflict); READ_NODE_FIELD(returningList); READ_NODE_FIELD(groupClause); READ_NODE_FIELD(havingQual); @@ -1131,6 +1132,22 @@ _readCurrentOfExpr(void) } /* + * _readInferenceElem + */ +static InferenceElem * +_readInferenceElem(void) +{ + READ_LOCALS(InferenceElem); + + READ_NODE_FIELD(expr); + READ_OID_FIELD(infercollid); + READ_OID_FIELD(inferopfamily); + READ_OID_FIELD(inferopcinputtype); + + READ_DONE(); +} + +/* * _readTargetEntry */ static TargetEntry * @@ -1196,6 +1213,25 @@ _readFromExpr(void) READ_DONE(); } +/* + * _readOnConflictExpr + */ +static OnConflictExpr * +_readOnConflictExpr(void) +{ + READ_LOCALS(OnConflictExpr); + + READ_ENUM_FIELD(action, OnConflictAction); + READ_NODE_FIELD(arbiterElems); + READ_NODE_FIELD(arbiterWhere); + READ_NODE_FIELD(onConflictSet); + READ_NODE_FIELD(onConflictWhere); + READ_OID_FIELD(constraint); + READ_INT_FIELD(exclRelIndex); + READ_NODE_FIELD(exclRelTlist); + + READ_DONE(); +} /* * Stuff from parsenodes.h. @@ -1395,6 +1431,8 @@ parseNodeString(void) return_value = _readSetToDefault(); else if (MATCH("CURRENTOFEXPR", 13)) return_value = _readCurrentOfExpr(); + else if (MATCH("INFERENCEELEM", 13)) + return_value = _readInferenceElem(); else if (MATCH("TARGETENTRY", 11)) return_value = _readTargetEntry(); else if (MATCH("RANGETBLREF", 11)) @@ -1403,6 +1441,8 @@ parseNodeString(void) return_value = _readJoinExpr(); else if (MATCH("FROMEXPR", 8)) return_value = _readFromExpr(); + else if (MATCH("ONCONFLICTEXPR", 14)) + return_value = _readOnConflictExpr(); else if (MATCH("RTE", 3)) return_value = _readRangeTblEntry(); else if (MATCH("RANGETBLFUNCTION", 16)) diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index eeb2a417643..3246332d6e3 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -4868,7 +4868,7 @@ make_modifytable(PlannerInfo *root, Index nominalRelation, List *resultRelations, List *subplans, List *withCheckOptionLists, List *returningLists, - List *rowMarks, int epqParam) + List *rowMarks, OnConflictExpr *onconflict, int epqParam) { ModifyTable *node = makeNode(ModifyTable); Plan *plan = &node->plan; @@ -4918,6 +4918,30 @@ make_modifytable(PlannerInfo *root, node->resultRelations = resultRelations; node->resultRelIndex = -1; /* will be set correctly in setrefs.c */ node->plans = subplans; + if (!onconflict) + { + node->onConflictAction = ONCONFLICT_NONE; + node->onConflictSet = NIL; + node->onConflictWhere = NULL; + node->arbiterIndexes = NIL; + } + else + { + node->onConflictAction = onconflict->action; + node->onConflictSet = onconflict->onConflictSet; + node->onConflictWhere = onconflict->onConflictWhere; + + /* + * If a set of unique index inference elements was provided (an + * INSERT...ON CONFLICT "inference specification"), then infer + * appropriate unique indexes (or throw an error if none are + * available). + */ + node->arbiterIndexes = infer_arbiter_indexes(root); + + node->exclRelRTI = onconflict->exclRelIndex; + node->exclRelTlist = onconflict->exclRelTlist; + } node->withCheckOptionLists = withCheckOptionLists; node->returningLists = returningLists; node->rowMarks = rowMarks; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index ea4d4c55cbd..c80d45acaa9 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -243,6 +243,8 @@ standard_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) result->queryId = parse->queryId; result->hasReturning = (parse->returningList != NIL); result->hasModifyingCTE = parse->hasModifyingCTE; + result->isUpsert = + (parse->onConflict && parse->onConflict->action == ONCONFLICT_UPDATE); result->canSetTag = parse->canSetTag; result->transientPlan = glob->transientPlan; result->planTree = top_plan; @@ -462,6 +464,17 @@ subquery_planner(PlannerGlobal *glob, Query *parse, parse->limitCount = preprocess_expression(root, parse->limitCount, EXPRKIND_LIMIT); + if (parse->onConflict) + { + parse->onConflict->onConflictSet = (List *) + preprocess_expression(root, (Node *) parse->onConflict->onConflictSet, + EXPRKIND_TARGET); + + parse->onConflict->onConflictWhere = + preprocess_expression(root, (Node *) parse->onConflict->onConflictWhere, + EXPRKIND_QUAL); + } + root->append_rel_list = (List *) preprocess_expression(root, (Node *) root->append_rel_list, EXPRKIND_APPINFO); @@ -612,6 +625,7 @@ subquery_planner(PlannerGlobal *glob, Query *parse, withCheckOptionLists, returningLists, rowMarks, + parse->onConflict, SS_assign_special_param(root)); } } @@ -802,6 +816,8 @@ inheritance_planner(PlannerInfo *root) List *rowMarks; ListCell *lc; + Assert(parse->commandType != CMD_INSERT); + /* * We generate a modified instance of the original Query for each target * relation, plan that, and put all the plans into a list that will be @@ -1046,6 +1062,8 @@ inheritance_planner(PlannerInfo *root) if (parse->returningList) returningLists = lappend(returningLists, subroot.parse->returningList); + + Assert(!parse->onConflict); } /* Mark result as unordered (probably unnecessary) */ @@ -1095,6 +1113,7 @@ inheritance_planner(PlannerInfo *root) withCheckOptionLists, returningLists, rowMarks, + NULL, SS_assign_special_param(root)); } @@ -1228,6 +1247,7 @@ grouping_planner(PlannerInfo *root, double tuple_fraction) bool use_hashed_grouping = false; WindowFuncLists *wflists = NULL; List *activeWindows = NIL; + OnConflictExpr *onconfl; MemSet(&agg_costs, 0, sizeof(AggClauseCosts)); @@ -1242,6 +1262,13 @@ grouping_planner(PlannerInfo *root, double tuple_fraction) /* Preprocess targetlist */ tlist = preprocess_targetlist(root, tlist); + onconfl = parse->onConflict; + if (onconfl) + onconfl->onConflictSet = + preprocess_onconflict_targetlist(onconfl->onConflictSet, + parse->resultRelation, + parse->rtable); + /* * Expand any rangetable entries that have security barrier quals. * This may add new security barrier subquery RTEs to the rangetable. diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index b7d6ff11223..612d32571af 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -739,7 +739,35 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) splan->plan.targetlist = copyObject(linitial(newRL)); } + /* + * We treat ModifyTable with ON CONFLICT as a form of 'pseudo + * join', where the inner side is the EXLUDED tuple. Therefore + * use fix_join_expr to setup the relevant variables to + * INNER_VAR. We explicitly don't create any OUTER_VARs as + * those are already used by RETURNING and it seems better to + * be non-conflicting. + */ + if (splan->onConflictSet) + { + indexed_tlist *itlist; + + itlist = build_tlist_index(splan->exclRelTlist); + + splan->onConflictSet = + fix_join_expr(root, splan->onConflictSet, + NULL, itlist, + linitial_int(splan->resultRelations), + rtoffset); + + splan->onConflictWhere = (Node *) + fix_join_expr(root, (List *) splan->onConflictWhere, + NULL, itlist, + linitial_int(splan->resultRelations), + rtoffset); + } + splan->nominalRelation += rtoffset; + splan->exclRelRTI += rtoffset; foreach(l, splan->resultRelations) { @@ -1846,7 +1874,8 @@ search_indexed_tlist_for_sortgroupref(Node *node, * inner_itlist = NULL and acceptable_rel = the ID of the target relation. * * 'clauses' is the targetlist or list of join clauses - * 'outer_itlist' is the indexed target list of the outer join relation + * 'outer_itlist' is the indexed target list of the outer join relation, + * or NULL * 'inner_itlist' is the indexed target list of the inner join relation, * or NULL * 'acceptable_rel' is either zero or the rangetable index of a relation @@ -1886,12 +1915,17 @@ fix_join_expr_mutator(Node *node, fix_join_expr_context *context) Var *var = (Var *) node; /* First look for the var in the input tlists */ - newvar = search_indexed_tlist_for_var(var, - context->outer_itlist, - OUTER_VAR, - context->rtoffset); - if (newvar) - return (Node *) newvar; + if (context->outer_itlist) + { + newvar = search_indexed_tlist_for_var(var, + context->outer_itlist, + OUTER_VAR, + context->rtoffset); + if (newvar) + return (Node *) newvar; + } + + /* Then in the outer */ if (context->inner_itlist) { newvar = search_indexed_tlist_for_var(var, @@ -1920,7 +1954,7 @@ fix_join_expr_mutator(Node *node, fix_join_expr_context *context) PlaceHolderVar *phv = (PlaceHolderVar *) node; /* See if the PlaceHolderVar has bubbled up from a lower plan node */ - if (context->outer_itlist->has_ph_vars) + if (context->outer_itlist && context->outer_itlist->has_ph_vars) { newvar = search_indexed_tlist_for_non_var((Node *) phv, context->outer_itlist, @@ -1943,7 +1977,7 @@ fix_join_expr_mutator(Node *node, fix_join_expr_context *context) if (IsA(node, Param)) return fix_param_node(context->root, (Param *) node); /* Try matching more complex expressions too, if tlists have any */ - if (context->outer_itlist->has_non_vars) + if (context->outer_itlist && context->outer_itlist->has_non_vars) { newvar = search_indexed_tlist_for_non_var(node, context->outer_itlist, diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index acfd0bcfbe5..0220672fc43 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -2340,6 +2340,10 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params, locally_added_param); finalize_primnode((Node *) mtplan->returningLists, &context); + finalize_primnode((Node *) mtplan->onConflictSet, + &context); + finalize_primnode((Node *) mtplan->onConflictWhere, + &context); foreach(l, mtplan->plans) { context.paramids = diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 50acfe40e97..4f0dc80d025 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -1030,6 +1030,9 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, pullup_replace_vars((Node *) parse->targetList, &rvcontext); parse->returningList = (List *) pullup_replace_vars((Node *) parse->returningList, &rvcontext); + if (parse->onConflict) + parse->onConflict->onConflictSet = (List *) + pullup_replace_vars((Node *) parse->onConflict->onConflictSet, &rvcontext); replace_vars_in_jointree((Node *) parse->jointree, &rvcontext, lowest_nulling_outer_join); Assert(parse->setOperations == NULL); @@ -1605,6 +1608,9 @@ pull_up_simple_values(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte) pullup_replace_vars((Node *) parse->targetList, &rvcontext); parse->returningList = (List *) pullup_replace_vars((Node *) parse->returningList, &rvcontext); + if (parse->onConflict) + parse->onConflict->onConflictSet = (List *) + pullup_replace_vars((Node *) parse->onConflict->onConflictSet, &rvcontext); replace_vars_in_jointree((Node *) parse->jointree, &rvcontext, NULL); Assert(parse->setOperations == NULL); parse->havingQual = pullup_replace_vars(parse->havingQual, &rvcontext); diff --git a/src/backend/optimizer/prep/preptlist.c b/src/backend/optimizer/prep/preptlist.c index 580c8467703..6b0c689e0c9 100644 --- a/src/backend/optimizer/prep/preptlist.c +++ b/src/backend/optimizer/prep/preptlist.c @@ -181,6 +181,19 @@ preprocess_targetlist(PlannerInfo *root, List *tlist) return tlist; } +/* + * preprocess_onconflict_targetlist + * Process ON CONFLICT SET targetlist. + * + * Returns the new targetlist. + */ +List * +preprocess_onconflict_targetlist(List *tlist, int result_relation, List *range_table) +{ + return expand_targetlist(tlist, CMD_UPDATE, result_relation, range_table); +} + + /***************************************************************************** * * TARGETLIST EXPANSION diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 068ab39dd43..8bcc5064a37 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -25,6 +25,7 @@ #include "access/transam.h" #include "access/xlog.h" #include "catalog/catalog.h" +#include "catalog/dependency.h" #include "catalog/heap.h" #include "foreign/fdwapi.h" #include "miscadmin.h" @@ -50,6 +51,8 @@ int constraint_exclusion = CONSTRAINT_EXCLUSION_PARTITION; get_relation_info_hook_type get_relation_info_hook = NULL; +static bool infer_collation_opclass_match(InferenceElem *elem, Relation idxRel, + Bitmapset *inferAttrs, List *idxExprs); static int32 get_rel_data_width(Relation rel, int32 *attr_widths); static List *get_relation_constraints(PlannerInfo *root, Oid relationObjectId, RelOptInfo *rel, @@ -400,6 +403,355 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, } /* + * infer_arbiter_indexes - + * Determine the unique indexes used to arbitrate speculative insertion. + * + * Uses user-supplied inference clause expressions and predicate to match a + * unique index from those defined and ready on the heap relation (target). + * An exact match is required on columns/expressions (although they can appear + * in any order). However, the predicate given by the user need only restrict + * insertion to a subset of some part of the table covered by some particular + * unique index (in particular, a partial unique index) in order to be + * inferred. + * + * The implementation does not consider which B-Tree operator class any + * particular available unique index attribute uses, unless one was specified + * in the inference specification. The same is true of collations. In + * particular, there is no system dependency on the default operator class for + * the purposes of inference. If no opclass (or collation) is specified, then + * all matching indexes (that may or may not match the default in terms of + * each attribute opclass/collation) are used for inference. + */ +List * +infer_arbiter_indexes(PlannerInfo *root) +{ + OnConflictExpr *onconflict = root->parse->onConflict; + /* Iteration state */ + Relation relation; + Oid relationObjectId; + Oid indexOidFromConstraint = InvalidOid; + List *indexList; + ListCell *l; + + /* Normalized inference attributes and inference expressions: */ + Bitmapset *inferAttrs = NULL; + List *inferElems = NIL; + + /* Result */ + List *candidates = NIL; + + /* + * Quickly return NIL for ON CONFLICT DO NOTHING without an inference + * specification or named constraint. ON CONFLICT DO UPDATE statements + * must always provide one or the other (but parser ought to have caught + * that already). + */ + if (onconflict->arbiterElems == NIL && + onconflict->constraint == InvalidOid) + return NIL; + + /* + * We need not lock the relation since it was already locked, either by + * the rewriter or when expand_inherited_rtentry() added it to the query's + * rangetable. + */ + relationObjectId = rt_fetch(root->parse->resultRelation, + root->parse->rtable)->relid; + + relation = heap_open(relationObjectId, NoLock); + + /* + * Build normalized/BMS representation of plain indexed attributes, as + * well as direct list of inference elements. This is required for + * matching the cataloged definition of indexes. + */ + foreach(l, onconflict->arbiterElems) + { + InferenceElem *elem; + Var *var; + int attno; + + elem = (InferenceElem *) lfirst(l); + + /* + * Parse analysis of inference elements performs full parse analysis + * of Vars, even for non-expression indexes (in contrast with utility + * command related use of IndexElem). However, indexes are cataloged + * with simple attribute numbers for non-expression indexes. Those + * are handled later. + */ + if (!IsA(elem->expr, Var)) + { + inferElems = lappend(inferElems, elem->expr); + continue; + } + + var = (Var *) elem->expr; + attno = var->varattno; + + if (attno < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), + errmsg("system columns cannot be used in an ON CONFLICT clause"))); + else if (attno == 0) + elog(ERROR, "whole row unique index inference specifications are not valid"); + + inferAttrs = bms_add_member(inferAttrs, attno); + } + + /* + * Lookup named constraint's index. This is not immediately returned + * because some additional sanity checks are required. + */ + if (onconflict->constraint != InvalidOid) + { + indexOidFromConstraint = get_constraint_index(onconflict->constraint); + + if (indexOidFromConstraint == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("constraint in ON CONFLICT clause has no associated index"))); + } + + indexList = RelationGetIndexList(relation); + + /* + * Using that representation, iterate through the list of indexes on the + * target relation to try and find a match + */ + foreach(l, indexList) + { + Oid indexoid = lfirst_oid(l); + Relation idxRel; + Form_pg_index idxForm; + Bitmapset *indexedAttrs = NULL; + List *idxExprs; + List *predExprs; + List *whereExplicit; + AttrNumber natt; + ListCell *el; + + /* + * Extract info from the relation descriptor for the index. We know + * that this is a target, so get lock type it is known will ultimately + * be required by the executor. + * + * Let executor complain about !indimmediate case directly, because + * enforcement needs to occur there anyway when an inference clause is + * omitted. + */ + idxRel = index_open(indexoid, RowExclusiveLock); + idxForm = idxRel->rd_index; + + if (!IndexIsValid(idxForm)) + goto next; + + /* + * If the index is valid, but cannot yet be used, ignore it. See + * src/backend/access/heap/README.HOT for discussion. + */ + if (idxForm->indcheckxmin && + !TransactionIdPrecedes(HeapTupleHeaderGetXmin(idxRel->rd_indextuple->t_data), + TransactionXmin)) + goto next; + + /* + * Look for match on "ON constraint_name" variant, which may not be + * unique constraint. This can only be a constraint name. + */ + if (indexOidFromConstraint == idxForm->indexrelid) + { + if (!idxForm->indisunique && onconflict->action == ONCONFLICT_UPDATE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("ON CONFLICT DO UPDATE not supported with exclusion constraints"))); + + list_free(indexList); + index_close(idxRel, NoLock); + heap_close(relation, NoLock); + candidates = lappend_oid(candidates, idxForm->indexrelid); + return candidates; + } + else if (indexOidFromConstraint != InvalidOid) + { + /* No point in further work for index in named constraint case */ + goto next; + } + + /* + * Only considering conventional inference at this point (not named + * constraints), so index under consideration can be immediately + * skipped if it's not unique + */ + if (!idxForm->indisunique) + goto next; + + /* Build BMS representation of cataloged index attributes */ + for (natt = 0; natt < idxForm->indnatts; natt++) + { + int attno = idxRel->rd_index->indkey.values[natt]; + + if (attno < 0) + elog(ERROR, "system column in index"); + + if (attno != 0) + indexedAttrs = bms_add_member(indexedAttrs, attno); + } + + /* Non-expression attributes (if any) must match */ + if (!bms_equal(indexedAttrs, inferAttrs)) + goto next; + + /* Expression attributes (if any) must match */ + idxExprs = RelationGetIndexExpressions(idxRel); + foreach(el, onconflict->arbiterElems) + { + InferenceElem *elem = (InferenceElem *) lfirst(el); + + /* + * Ensure that collation/opclass aspects of inference expression + * element match. Even though this loop is primarily concerned + * with matching expressions, it is a convenient point to check + * this for both expressions and ordinary (non-expression) + * attributes appearing as inference elements. + */ + if (!infer_collation_opclass_match(elem, idxRel, inferAttrs, + idxExprs)) + goto next; + + /* + * Plain Vars don't factor into count of expression elements, and + * the question of whether or not they satisfy the index + * definition has already been considered (they must). + */ + if (IsA(elem->expr, Var)) + continue; + + /* + * Might as well avoid redundant check in the rare cases where + * infer_collation_opclass_match() is required to do real work. + * Otherwise, check that element expression appears in cataloged + * index definition. + */ + if (elem->infercollid != InvalidOid || + elem->inferopfamily != InvalidOid || + list_member(idxExprs, elem->expr)) + continue; + + goto next; + } + + /* + * Now that all inference elements were matched, ensure that the + * expression elements from inference clause are not missing any + * cataloged expressions. This does the right thing when unique + * indexes redundantly repeat the same attribute, or if attributes + * redundantly appear multiple times within an inference clause. + */ + if (list_difference(idxExprs, inferElems) != NIL) + goto next; + + /* + * Any user-supplied ON CONFLICT unique index inference WHERE clause + * need only be implied by the cataloged index definitions predicate. + */ + predExprs = RelationGetIndexPredicate(idxRel); + whereExplicit = make_ands_implicit((Expr *) onconflict->arbiterWhere); + + if (!predicate_implied_by(predExprs, whereExplicit)) + goto next; + + candidates = lappend_oid(candidates, idxForm->indexrelid); +next: + index_close(idxRel, NoLock); + } + + list_free(indexList); + heap_close(relation, NoLock); + + if (candidates == NIL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), + errmsg("there is no unique or exclusion constraint matching the ON CONFLICT specification"))); + + return candidates; +} + +/* + * infer_collation_opclass_match - ensure infer element opclass/collation match + * + * Given unique index inference element from inference specification, if + * collation was specified, or if opclass (represented here as opfamily + + * opcintype) was specified, verify that there is at least one matching + * indexed attribute (occasionally, there may be more). Skip this in the + * common case where inference specification does not include collation or + * opclass (instead matching everything, regardless of cataloged + * collation/opclass of indexed attribute). + * + * At least historically, Postgres has not offered collations or opclasses + * with alternative-to-default notions of equality, so these additional + * criteria should only be required infrequently. + * + * Don't give up immediately when an inference element matches some attribute + * cataloged as indexed but not matching additional opclass/collation + * criteria. This is done so that the implementation is as forgiving as + * possible of redundancy within cataloged index attributes (or, less + * usefully, within inference specification elements). If collations actually + * differ between apparently redundantly indexed attributes (redundant within + * or across indexes), then there really is no redundancy as such. + * + * Note that if an inference element specifies an opclass and a collation at + * once, both must match in at least one particular attribute within index + * catalog definition in order for that inference element to be considered + * inferred/satisfied. + */ +static bool +infer_collation_opclass_match(InferenceElem *elem, Relation idxRel, + Bitmapset *inferAttrs, List *idxExprs) +{ + AttrNumber natt; + + /* + * If inference specification element lacks collation/opclass, then no + * need to check for exact match. + */ + if (elem->infercollid == InvalidOid && elem->inferopfamily == InvalidOid) + return true; + + for (natt = 1; natt <= idxRel->rd_att->natts; natt++) + { + Oid opfamily = idxRel->rd_opfamily[natt - 1]; + Oid opcinputtype = idxRel->rd_opcintype[natt - 1]; + Oid collation = idxRel->rd_indcollation[natt - 1]; + + if (elem->inferopfamily != InvalidOid && + (elem->inferopfamily != opfamily || + elem->inferopcinputtype != opcinputtype)) + { + /* Attribute needed to match opclass, but didn't */ + continue; + } + + if (elem->infercollid != InvalidOid && + elem->infercollid != collation) + { + /* Attribute needed to match collation, but didn't */ + continue; + } + + if ((IsA(elem->expr, Var) && + bms_is_member(((Var *) elem->expr)->varattno, inferAttrs)) || + list_member(idxExprs, elem->expr)) + { + /* Found one match - good enough */ + return true; + } + } + + return false; +} + +/* * estimate_rel_size - estimate # pages and # tuples in a table or index * * We also estimate the fraction of the pages that are marked all-visible in diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index 2d320d100b8..3eb4feabfd6 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -52,6 +52,8 @@ static Query *transformDeleteStmt(ParseState *pstate, DeleteStmt *stmt); static Query *transformInsertStmt(ParseState *pstate, InsertStmt *stmt); static List *transformInsertRow(ParseState *pstate, List *exprlist, List *stmtcols, List *icolumns, List *attrnos); +static OnConflictExpr *transformOnConflictClause(ParseState *pstate, + OnConflictClause *onConflictClause); static int count_rowexpr_columns(ParseState *pstate, Node *expr); static Query *transformSelectStmt(ParseState *pstate, SelectStmt *stmt); static Query *transformValuesClause(ParseState *pstate, SelectStmt *stmt); @@ -62,6 +64,8 @@ static void determineRecursiveColTypes(ParseState *pstate, Node *larg, List *nrtargetlist); static Query *transformUpdateStmt(ParseState *pstate, UpdateStmt *stmt); static List *transformReturningList(ParseState *pstate, List *returningList); +static List *transformUpdateTargetList(ParseState *pstate, + List *targetList); static Query *transformDeclareCursorStmt(ParseState *pstate, DeclareCursorStmt *stmt); static Query *transformExplainStmt(ParseState *pstate, @@ -419,6 +423,8 @@ transformInsertStmt(ParseState *pstate, InsertStmt *stmt) ListCell *icols; ListCell *attnos; ListCell *lc; + bool isOnConflictUpdate; + AclMode targetPerms; /* There can't be any outer WITH to worry about */ Assert(pstate->p_ctenamespace == NIL); @@ -434,6 +440,9 @@ transformInsertStmt(ParseState *pstate, InsertStmt *stmt) qry->hasModifyingCTE = pstate->p_hasModifyingCTE; } + isOnConflictUpdate = (stmt->onConflictClause && + stmt->onConflictClause->action == ONCONFLICT_UPDATE); + /* * We have three cases to deal with: DEFAULT VALUES (selectStmt == NULL), * VALUES list, or general SELECT input. We special-case VALUES, both for @@ -478,8 +487,11 @@ transformInsertStmt(ParseState *pstate, InsertStmt *stmt) * mentioned in the SELECT part. Note that the target table is not added * to the joinlist or namespace. */ + targetPerms = ACL_INSERT; + if (isOnConflictUpdate) + targetPerms |= ACL_UPDATE; qry->resultRelation = setTargetTable(pstate, stmt->relation, - false, false, ACL_INSERT); + false, false, targetPerms); /* Validate stmt->cols list, or build default list if no list given */ icolumns = checkInsertTargets(pstate, stmt->cols, &attrnos); @@ -740,6 +752,11 @@ transformInsertStmt(ParseState *pstate, InsertStmt *stmt) attnos = lnext(attnos); } + /* Process ON CONFLICT, if any. */ + if (stmt->onConflictClause) + qry->onConflict = transformOnConflictClause(pstate, + stmt->onConflictClause); + /* * If we have a RETURNING clause, we need to add the target relation to * the query namespace before processing it, so that Var references in @@ -850,6 +867,85 @@ transformInsertRow(ParseState *pstate, List *exprlist, } /* + * transformSelectStmt - + * transforms an OnConflictClause in an INSERT + */ +static OnConflictExpr * +transformOnConflictClause(ParseState *pstate, + OnConflictClause *onConflictClause) +{ + List *arbiterElems; + Node *arbiterWhere; + Oid arbiterConstraint; + List *onConflictSet = NIL; + Node *onConflictWhere = NULL; + RangeTblEntry *exclRte = NULL; + int exclRelIndex = 0; + List *exclRelTlist = NIL; + OnConflictExpr *result; + + /* Process the arbiter clause, ON CONFLICT ON (...) */ + transformOnConflictArbiter(pstate, onConflictClause, &arbiterElems, + &arbiterWhere, &arbiterConstraint); + + /* Process DO UPDATE */ + if (onConflictClause->action == ONCONFLICT_UPDATE) + { + exclRte = addRangeTableEntryForRelation(pstate, + pstate->p_target_relation, + makeAlias("excluded", NIL), + false, false); + exclRelIndex = list_length(pstate->p_rtable); + + /* + * Build a targetlist for the EXCLUDED pseudo relation. Out of + * simplicity we do that here, because expandRelAttrs() happens to + * nearly do the right thing; specifically it also works with views. + * It'd be more proper to instead scan some pseudo scan node, but it + * doesn't seem worth the amount of code required. + * + * The only caveat of this hack is that the permissions expandRelAttrs + * adds have to be reset. markVarForSelectPriv() will add the exact + * required permissions back. + */ + exclRelTlist = expandRelAttrs(pstate, exclRte, + exclRelIndex, 0, -1); + exclRte->requiredPerms = 0; + exclRte->selectedCols = NULL; + + /* + * Add EXCLUDED and the target RTE to the namespace, so that they can + * be used in the UPDATE statement. + */ + addRTEtoQuery(pstate, exclRte, false, true, true); + addRTEtoQuery(pstate, pstate->p_target_rangetblentry, + false, true, true); + + onConflictSet = + transformUpdateTargetList(pstate, onConflictClause->targetList); + + onConflictWhere = transformWhereClause(pstate, + onConflictClause->whereClause, + EXPR_KIND_WHERE, "WHERE"); + } + + /* Finally, build ON CONFLICT DO [NOTHING | UPDATE] expression */ + result = makeNode(OnConflictExpr); + + result->action = onConflictClause->action; + result->arbiterElems = arbiterElems; + result->arbiterWhere = arbiterWhere; + result->constraint = arbiterConstraint; + result->onConflictSet = onConflictSet; + result->onConflictWhere = onConflictWhere; + result->exclRelIndex = exclRelIndex; + result->exclRelTlist = exclRelTlist; + + return result; +} + + +/* * count_rowexpr_columns - * get number of columns contained in a ROW() expression; * return -1 if expression isn't a RowExpr or a Var referencing one. @@ -1899,10 +1995,7 @@ transformUpdateStmt(ParseState *pstate, UpdateStmt *stmt) { Query *qry = makeNode(Query); ParseNamespaceItem *nsitem; - RangeTblEntry *target_rte; Node *qual; - ListCell *origTargetList; - ListCell *tl; qry->commandType = CMD_UPDATE; pstate->p_is_update = true; @@ -1937,23 +2030,41 @@ transformUpdateStmt(ParseState *pstate, UpdateStmt *stmt) nsitem->p_lateral_only = false; nsitem->p_lateral_ok = true; - qry->targetList = transformTargetList(pstate, stmt->targetList, - EXPR_KIND_UPDATE_SOURCE); - qual = transformWhereClause(pstate, stmt->whereClause, EXPR_KIND_WHERE, "WHERE"); qry->returningList = transformReturningList(pstate, stmt->returningList); + /* + * Now we are done with SELECT-like processing, and can get on with + * transforming the target list to match the UPDATE target columns. + */ + qry->targetList = transformUpdateTargetList(pstate, stmt->targetList); + qry->rtable = pstate->p_rtable; qry->jointree = makeFromExpr(pstate->p_joinlist, qual); qry->hasSubLinks = pstate->p_hasSubLinks; - /* - * Now we are done with SELECT-like processing, and can get on with - * transforming the target list to match the UPDATE target columns. - */ + assign_query_collations(pstate, qry); + + return qry; +} + +/* + * transformUpdateTargetList - + * handle SET clause in UPDATE/INSERT ... ON CONFLICT UPDATE + */ +static List * +transformUpdateTargetList(ParseState *pstate, List *origTlist) +{ + List *tlist = NIL; + RangeTblEntry *target_rte; + ListCell *orig_tl; + ListCell *tl; + + tlist = transformTargetList(pstate, origTlist, + EXPR_KIND_UPDATE_SOURCE); /* Prepare to assign non-conflicting resnos to resjunk attributes */ if (pstate->p_next_resno <= pstate->p_target_relation->rd_rel->relnatts) @@ -1961,9 +2072,9 @@ transformUpdateStmt(ParseState *pstate, UpdateStmt *stmt) /* Prepare non-junk columns for assignment to target table */ target_rte = pstate->p_target_rangetblentry; - origTargetList = list_head(stmt->targetList); + orig_tl = list_head(origTlist); - foreach(tl, qry->targetList) + foreach(tl, tlist) { TargetEntry *tle = (TargetEntry *) lfirst(tl); ResTarget *origTarget; @@ -1981,9 +2092,9 @@ transformUpdateStmt(ParseState *pstate, UpdateStmt *stmt) tle->resname = NULL; continue; } - if (origTargetList == NULL) + if (orig_tl == NULL) elog(ERROR, "UPDATE target count mismatch --- internal error"); - origTarget = (ResTarget *) lfirst(origTargetList); + origTarget = (ResTarget *) lfirst(orig_tl); Assert(IsA(origTarget, ResTarget)); attrno = attnameAttNum(pstate->p_target_relation, @@ -2005,14 +2116,12 @@ transformUpdateStmt(ParseState *pstate, UpdateStmt *stmt) target_rte->updatedCols = bms_add_member(target_rte->updatedCols, attrno - FirstLowInvalidHeapAttributeNumber); - origTargetList = lnext(origTargetList); + orig_tl = lnext(orig_tl); } - if (origTargetList != NULL) + if (orig_tl != NULL) elog(ERROR, "UPDATE target count mismatch --- internal error"); - assign_query_collations(pstate, qry); - - return qry; + return tlist; } /* diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 0180530a309..7a4c07365c1 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -217,6 +217,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); RangeVar *range; IntoClause *into; WithClause *with; + InferClause *infer; + OnConflictClause *onconflict; A_Indices *aind; ResTarget *target; struct PrivTarget *privtarget; @@ -318,7 +320,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); opt_class opt_inline_handler opt_validator validator_clause opt_collate -%type <range> qualified_name OptConstrFromTable +%type <range> qualified_name insert_target OptConstrFromTable %type <str> all_Op MathOp @@ -344,7 +346,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); OptTableElementList TableElementList OptInherit definition OptTypedTableElementList TypedTableElementList reloptions opt_reloptions - OptWith opt_distinct opt_definition func_args func_args_list + OptWith distinct_clause opt_all_clause opt_definition func_args func_args_list func_args_with_defaults func_args_with_defaults_list aggr_args aggr_args_list func_as createfunc_opt_list alterfunc_opt_list @@ -389,7 +391,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type <node> for_locking_item %type <list> for_locking_clause opt_for_locking_clause for_locking_items %type <list> locked_rels_list -%type <boolean> opt_all +%type <boolean> all_or_distinct %type <node> join_outer join_qual %type <jtype> join_type @@ -418,6 +420,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type <defelt> SeqOptElem %type <istmt> insert_rest +%type <infer> opt_conf_expr +%type <onconflict> opt_on_conflict %type <vsetstmt> generic_set set_rest set_rest_more generic_reset reset_rest SetResetClause FunctionSetResetClause @@ -557,8 +561,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); CACHE CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE CLUSTER COALESCE COLLATE COLLATION COLUMN COMMENT COMMENTS COMMIT - COMMITTED CONCURRENTLY CONFIGURATION CONNECTION CONSTRAINT CONSTRAINTS - CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE + COMMITTED CONCURRENTLY CONFIGURATION CONFLICT CONNECTION CONSTRAINT + CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE CROSS CSV CURRENT_P CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE @@ -9436,15 +9440,35 @@ DeallocateStmt: DEALLOCATE name *****************************************************************************/ InsertStmt: - opt_with_clause INSERT INTO qualified_name insert_rest returning_clause + opt_with_clause INSERT INTO insert_target insert_rest + opt_on_conflict returning_clause { $5->relation = $4; - $5->returningList = $6; + $5->onConflictClause = $6; + $5->returningList = $7; $5->withClause = $1; $$ = (Node *) $5; } ; +/* + * Can't easily make AS optional here, because VALUES in insert_rest would + * have a shift/reduce conflict with a values as a optional alias. We could + * easily allow unreserved_keywords as optional aliases, but that'd be a odd + * divergance from other places. So just require AS for now. + */ +insert_target: + qualified_name + { + $$ = $1; + } + | qualified_name AS ColId + { + $1->alias = makeAlias($3, NIL); + $$ = $1; + } + ; + insert_rest: SelectStmt { @@ -9484,6 +9508,56 @@ insert_column_item: } ; +opt_on_conflict: + ON CONFLICT opt_conf_expr DO UPDATE SET set_clause_list where_clause + { + $$ = makeNode(OnConflictClause); + $$->action = ONCONFLICT_UPDATE; + $$->infer = $3; + $$->targetList = $7; + $$->whereClause = $8; + $$->location = @1; + } + | + ON CONFLICT opt_conf_expr DO NOTHING + { + $$ = makeNode(OnConflictClause); + $$->action = ONCONFLICT_NOTHING; + $$->infer = $3; + $$->targetList = NIL; + $$->whereClause = NULL; + $$->location = @1; + } + | /*EMPTY*/ + { + $$ = NULL; + } + ; + +opt_conf_expr: + '(' index_params ')' where_clause + { + $$ = makeNode(InferClause); + $$->indexElems = $2; + $$->whereClause = $4; + $$->conname = NULL; + $$->location = @1; + } + | + ON CONSTRAINT name + { + $$ = makeNode(InferClause); + $$->indexElems = NIL; + $$->whereClause = NULL; + $$->conname = $3; + $$->location = @1; + } + | /*EMPTY*/ + { + $$ = NULL; + } + ; + returning_clause: RETURNING target_list { $$ = $2; } | /* EMPTY */ { $$ = NIL; } @@ -9870,7 +9944,21 @@ select_clause: * However, this is not checked by the grammar; parse analysis must check it. */ simple_select: - SELECT opt_distinct opt_target_list + SELECT opt_all_clause opt_target_list + into_clause from_clause where_clause + group_clause having_clause window_clause + { + SelectStmt *n = makeNode(SelectStmt); + n->targetList = $3; + n->intoClause = $4; + n->fromClause = $5; + n->whereClause = $6; + n->groupClause = $7; + n->havingClause = $8; + n->windowClause = $9; + $$ = (Node *)n; + } + | SELECT distinct_clause target_list into_clause from_clause where_clause group_clause having_clause window_clause { @@ -9905,15 +9993,15 @@ simple_select: n->fromClause = list_make1($2); $$ = (Node *)n; } - | select_clause UNION opt_all select_clause + | select_clause UNION all_or_distinct select_clause { $$ = makeSetOp(SETOP_UNION, $3, $1, $4); } - | select_clause INTERSECT opt_all select_clause + | select_clause INTERSECT all_or_distinct select_clause { $$ = makeSetOp(SETOP_INTERSECT, $3, $1, $4); } - | select_clause EXCEPT opt_all select_clause + | select_clause EXCEPT all_or_distinct select_clause { $$ = makeSetOp(SETOP_EXCEPT, $3, $1, $4); } @@ -10052,7 +10140,8 @@ opt_table: TABLE {} | /*EMPTY*/ {} ; -opt_all: ALL { $$ = TRUE; } +all_or_distinct: + ALL { $$ = TRUE; } | DISTINCT { $$ = FALSE; } | /*EMPTY*/ { $$ = FALSE; } ; @@ -10060,10 +10149,13 @@ opt_all: ALL { $$ = TRUE; } /* We use (NIL) as a placeholder to indicate that all target expressions * should be placed in the DISTINCT list during parsetree analysis. */ -opt_distinct: +distinct_clause: DISTINCT { $$ = list_make1(NIL); } | DISTINCT ON '(' expr_list ')' { $$ = $4; } - | ALL { $$ = NIL; } + ; + +opt_all_clause: + ALL { $$ = NIL;} | /*EMPTY*/ { $$ = NIL; } ; @@ -13367,6 +13459,7 @@ unreserved_keyword: | COMMIT | COMMITTED | CONFIGURATION + | CONFLICT | CONNECTION | CONSTRAINTS | CONTENT_P diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c index 8d90b5098a1..73c505ed85b 100644 --- a/src/backend/parser/parse_clause.c +++ b/src/backend/parser/parse_clause.c @@ -16,7 +16,9 @@ #include "postgres.h" #include "access/heapam.h" +#include "catalog/catalog.h" #include "catalog/heap.h" +#include "catalog/pg_constraint.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "nodes/makefuncs.h" @@ -32,6 +34,7 @@ #include "parser/parse_oper.h" #include "parser/parse_relation.h" #include "parser/parse_target.h" +#include "parser/parse_type.h" #include "rewrite/rewriteManip.h" #include "utils/guc.h" #include "utils/lsyscache.h" @@ -75,6 +78,8 @@ static TargetEntry *findTargetlistEntrySQL99(ParseState *pstate, Node *node, List **tlist, ParseExprKind exprKind); static int get_matching_location(int sortgroupref, List *sortgrouprefs, List *exprs); +static List *resolve_unique_index_expr(ParseState *pstate, InferClause * infer, + Relation heapRel); static List *addTargetToGroupList(ParseState *pstate, TargetEntry *tle, List *grouplist, List *targetlist, int location, bool resolveUnknown); @@ -2167,6 +2172,204 @@ get_matching_location(int sortgroupref, List *sortgrouprefs, List *exprs) } /* + * resolve_unique_index_expr + * Infer a unique index from a list of indexElems, for ON + * CONFLICT clause + * + * Perform parse analysis of expressions and columns appearing within ON + * CONFLICT clause. During planning, the returned list of expressions is used + * to infer which unique index to use. + */ +static List * +resolve_unique_index_expr(ParseState *pstate, InferClause *infer, + Relation heapRel) +{ + List *result = NIL; + ListCell *l; + + foreach(l, infer->indexElems) + { + IndexElem *ielem = (IndexElem *) lfirst(l); + InferenceElem *pInfer = makeNode(InferenceElem); + Node *parse; + + /* + * Raw grammar re-uses CREATE INDEX infrastructure for unique index + * inference clause, and so will accept opclasses by name and so on. + * + * Make no attempt to match ASC or DESC ordering or NULLS FIRST/NULLS + * LAST ordering, since those are not significant for inference + * purposes (any unique index matching the inference specification in + * other regards is accepted indifferently). Actively reject this as + * wrong-headed. + */ + if (ielem->ordering != SORTBY_DEFAULT) + ereport(ERROR, + (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), + errmsg("ASC/DESC is not allowed in ON CONFLICT clause"), + parser_errposition(pstate, + exprLocation((Node *) infer)))); + if (ielem->nulls_ordering != SORTBY_NULLS_DEFAULT) + ereport(ERROR, + (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), + errmsg("NULLS FIRST/LAST is not allowed in ON CONFLICT clause"), + parser_errposition(pstate, + exprLocation((Node *) infer)))); + + if (!ielem->expr) + { + /* Simple index attribute */ + ColumnRef *n; + + /* + * Grammar won't have built raw expression for us in event of + * plain column reference. Create one directly, and perform + * expression transformation. Planner expects this, and performs + * its own normalization for the purposes of matching against + * pg_index. + */ + n = makeNode(ColumnRef); + n->fields = list_make1(makeString(ielem->name)); + /* Location is approximately that of inference specification */ + n->location = infer->location; + parse = (Node *) n; + } + else + { + /* Do parse transformation of the raw expression */ + parse = (Node *) ielem->expr; + } + + /* + * transformExpr() should have already rejected subqueries, + * aggregates, and window functions, based on the EXPR_KIND_ for an + * index expression. Expressions returning sets won't have been + * rejected, but don't bother doing so here; there should be no + * available expression unique index to match any such expression + * against anyway. + */ + pInfer->expr = transformExpr(pstate, parse, EXPR_KIND_INDEX_EXPRESSION); + + /* Perform lookup of collation and operator class as required */ + if (!ielem->collation) + pInfer->infercollid = InvalidOid; + else + pInfer->infercollid = LookupCollation(pstate, ielem->collation, + exprLocation(pInfer->expr)); + + if (!ielem->opclass) + { + pInfer->inferopfamily = InvalidOid; + pInfer->inferopcinputtype = InvalidOid; + } + else + { + Oid opclass = get_opclass_oid(BTREE_AM_OID, ielem->opclass, + false); + + pInfer->inferopfamily = get_opclass_family(opclass); + pInfer->inferopcinputtype = get_opclass_input_type(opclass); + } + + result = lappend(result, pInfer); + } + + return result; +} + +/* + * transformOnConflictArbiter - + * transform arbiter expressions in an ON CONFLICT clause. + * + * Transformed expressions used to infer one unique index relation to serve as + * an ON CONFLICT arbiter. Partial unique indexes may be inferred using WHERE + * clause from inference specification clause. + */ +void +transformOnConflictArbiter(ParseState *pstate, + OnConflictClause *onConflictClause, + List **arbiterExpr, Node **arbiterWhere, + Oid *constraint) +{ + InferClause *infer = onConflictClause->infer; + + *arbiterExpr = NIL; + *arbiterWhere = NULL; + *constraint = InvalidOid; + + if (onConflictClause->action == ONCONFLICT_UPDATE && !infer) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("ON CONFLICT DO UPDATE requires inference specification or constraint name"), + errhint("For example, ON CONFLICT ON CONFLICT (<column>)."), + parser_errposition(pstate, + exprLocation((Node *) onConflictClause)))); + + /* + * To simplify certain aspects of its design, speculative insertion into + * system catalogs is disallowed + */ + if (IsCatalogRelation(pstate->p_target_relation)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ON CONFLICT not supported with system catalog tables"), + parser_errposition(pstate, + exprLocation((Node *) onConflictClause)))); + + /* Same applies to table used by logical decoding as catalog table */ + if (RelationIsUsedAsCatalogTable(pstate->p_target_relation)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ON CONFLICT not supported on table \"%s\" used as a catalog table", + RelationGetRelationName(pstate->p_target_relation)), + parser_errposition(pstate, + exprLocation((Node *) onConflictClause)))); + + /* ON CONFLICT DO NOTHING does not require an inference clause */ + if (infer) + { + List *save_namespace; + + /* + * While we process the arbiter expressions, accept only + * non-qualified references to the target table. Hide any other + * relations. + */ + save_namespace = pstate->p_namespace; + pstate->p_namespace = NIL; + addRTEtoQuery(pstate, pstate->p_target_rangetblentry, + false, false, true); + + if (infer->indexElems) + *arbiterExpr = resolve_unique_index_expr(pstate, infer, + pstate->p_target_relation); + + /* + * Handling inference WHERE clause (for partial unique index + * inference) + */ + if (infer->whereClause) + *arbiterWhere = transformExpr(pstate, infer->whereClause, + EXPR_KIND_INDEX_PREDICATE); + + pstate->p_namespace = save_namespace; + + if (infer->conname) + *constraint = get_relation_constraint_oid(RelationGetRelid(pstate->p_target_relation), + infer->conname, false); + } + + /* + * It's convenient to form a list of expressions based on the + * representation used by CREATE INDEX, since the same restrictions are + * appropriate (e.g. on subqueries). However, from here on, a dedicated + * primnode representation is used for inference elements, and so + * assign_query_collations() can be trusted to do the right thing with the + * post parse analysis query tree inference clause representation. + */ +} + +/* * addTargetToSortList * If the given targetlist entry isn't already in the SortGroupClause * list, add it to the end of the list, using the given sort ordering diff --git a/src/backend/parser/parse_collate.c b/src/backend/parser/parse_collate.c index 7c6a11c7575..4c85b708d3b 100644 --- a/src/backend/parser/parse_collate.c +++ b/src/backend/parser/parse_collate.c @@ -479,9 +479,11 @@ assign_collations_walker(Node *node, assign_collations_context *context) parser_errposition(context->pstate, loccontext.location2))); break; + case T_InferenceElem: case T_RangeTblRef: case T_JoinExpr: case T_FromExpr: + case T_OnConflictExpr: case T_SortGroupClause: (void) expression_tree_walker(node, assign_collations_walker, diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c index 2d85cf08e70..59973ba9c3c 100644 --- a/src/backend/parser/parse_target.c +++ b/src/backend/parser/parse_target.c @@ -537,11 +537,12 @@ transformAssignedExpr(ParseState *pstate, /* * updateTargetListEntry() - * This is used in UPDATE statements only. It prepares an UPDATE - * TargetEntry for assignment to a column of the target table. - * This includes coercing the given value to the target column's type - * (if necessary), and dealing with any subfield names or subscripts - * attached to the target column itself. + * This is used in UPDATE statements (and ON CONFLICT DO UPDATE) + * only. It prepares an UPDATE TargetEntry for assignment to a + * column of the target table. This includes coercing the given + * value to the target column's type (if necessary), and dealing with + * any subfield names or subscripts attached to the target column + * itself. * * pstate parse state * tle target list entry to be modified diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c index 88424964ef3..ea388182692 100644 --- a/src/backend/replication/logical/decode.c +++ b/src/backend/replication/logical/decode.c @@ -64,6 +64,8 @@ static void DecodeInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); static void DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); static void DecodeDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); static void DecodeMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); +static void DecodeSpecConfirm(LogicalDecodingContext *ctx, XLogRecordBuffer *buf); + static void DecodeCommit(LogicalDecodingContext *ctx, XLogRecordBuffer *buf, xl_xact_parsed_commit *parsed, TransactionId xid); static void DecodeAbort(LogicalDecodingContext *ctx, XLogRecordBuffer *buf, @@ -414,6 +416,11 @@ DecodeHeapOp(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) ReorderBufferXidSetCatalogChanges(ctx->reorder, xid, buf->origptr); break; + case XLOG_HEAP_CONFIRM: + if (SnapBuildProcessChange(builder, xid, buf->origptr)) + DecodeSpecConfirm(ctx, buf); + break; + case XLOG_HEAP_LOCK: /* we don't care about row level locks for now */ break; @@ -564,11 +571,15 @@ DecodeInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) return; change = ReorderBufferGetChange(ctx->reorder); - change->action = REORDER_BUFFER_CHANGE_INSERT; + if (!(xlrec->flags & XLH_INSERT_IS_SPECULATIVE)) + change->action = REORDER_BUFFER_CHANGE_INSERT; + else + change->action = REORDER_BUFFER_CHANGE_INTERNAL_SPEC_INSERT; change->origin_id = XLogRecGetOrigin(r); + memcpy(&change->data.tp.relnode, &target_node, sizeof(RelFileNode)); - if (xlrec->flags & XLOG_HEAP_CONTAINS_NEW_TUPLE) + if (xlrec->flags & XLH_INSERT_CONTAINS_NEW_TUPLE) { Size tuplelen; char *tupledata = XLogRecGetBlockData(r, 0, &tuplelen); @@ -615,7 +626,7 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) change->origin_id = XLogRecGetOrigin(r); memcpy(&change->data.tp.relnode, &target_node, sizeof(RelFileNode)); - if (xlrec->flags & XLOG_HEAP_CONTAINS_NEW_TUPLE) + if (xlrec->flags & XLH_UPDATE_CONTAINS_NEW_TUPLE) { data = XLogRecGetBlockData(r, 0, &datalen); @@ -624,7 +635,7 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) DecodeXLogTuple(data, datalen, change->data.tp.newtuple); } - if (xlrec->flags & XLOG_HEAP_CONTAINS_OLD) + if (xlrec->flags & XLH_UPDATE_CONTAINS_OLD) { /* caution, remaining data in record is not aligned */ data = XLogRecGetData(r) + SizeOfHeapUpdate; @@ -660,6 +671,13 @@ DecodeDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) if (target_node.dbNode != ctx->slot->data.database) return; + /* + * Super deletions are irrelevant for logical decoding, it's driven by the + * confirmation records. + */ + if (xlrec->flags & XLH_DELETE_IS_SUPER) + return; + /* output plugin doesn't look for this origin, no need to queue */ if (FilterByOrigin(ctx, XLogRecGetOrigin(r))) return; @@ -671,7 +689,7 @@ DecodeDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) memcpy(&change->data.tp.relnode, &target_node, sizeof(RelFileNode)); /* old primary key stored */ - if (xlrec->flags & XLOG_HEAP_CONTAINS_OLD) + if (xlrec->flags & XLH_DELETE_CONTAINS_OLD) { Assert(XLogRecGetDataLen(r) > (SizeOfHeapDelete + SizeOfHeapHeader)); @@ -737,7 +755,7 @@ DecodeMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) * We decode the tuple in pretty much the same way as DecodeXLogTuple, * but since the layout is slightly different, we can't use it here. */ - if (xlrec->flags & XLOG_HEAP_CONTAINS_NEW_TUPLE) + if (xlrec->flags & XLH_INSERT_CONTAINS_NEW_TUPLE) { change->data.tp.newtuple = ReorderBufferGetTupleBuf(ctx->reorder); @@ -775,7 +793,7 @@ DecodeMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) * xl_multi_insert_tuple record emitted by one heap_multi_insert() * call. */ - if (xlrec->flags & XLOG_HEAP_LAST_MULTI_INSERT && + if (xlrec->flags & XLH_INSERT_LAST_IN_MULTI && (i + 1) == xlrec->ntuples) change->data.tp.clear_toast_afterwards = true; else @@ -788,6 +806,40 @@ DecodeMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) } /* + * Parse XLOG_HEAP_CONFIRM from wal into a confirmation change. + * + * This is pretty trivial, all the state essentially already setup by the + * speculative insertion. + */ +static void +DecodeSpecConfirm(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) +{ + XLogReaderState *r = buf->record; + ReorderBufferChange *change; + RelFileNode target_node; + + /* only interested in our database */ + XLogRecGetBlockTag(r, 0, &target_node, NULL, NULL); + if (target_node.dbNode != ctx->slot->data.database) + return; + + /* output plugin doesn't look for this origin, no need to queue */ + if (FilterByOrigin(ctx, XLogRecGetOrigin(r))) + return; + + change = ReorderBufferGetChange(ctx->reorder); + change->action = REORDER_BUFFER_CHANGE_INTERNAL_SPEC_CONFIRM; + change->origin_id = XLogRecGetOrigin(r); + + memcpy(&change->data.tp.relnode, &target_node, sizeof(RelFileNode)); + + change->data.tp.clear_toast_afterwards = true; + + ReorderBufferQueueChange(ctx->reorder, XLogRecGetXid(r), buf->origptr, change); +} + + +/* * Read a HeapTuple as WAL logged by heap_insert, heap_update and heap_delete * (but not by heap_multi_insert) into a tuplebuf. * diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index c9c1d1036e0..57854b0aa57 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -401,6 +401,7 @@ ReorderBufferReturnChange(ReorderBuffer *rb, ReorderBufferChange *change) case REORDER_BUFFER_CHANGE_INSERT: case REORDER_BUFFER_CHANGE_UPDATE: case REORDER_BUFFER_CHANGE_DELETE: + case REORDER_BUFFER_CHANGE_INTERNAL_SPEC_INSERT: if (change->data.tp.newtuple) { ReorderBufferReturnTupleBuf(rb, change->data.tp.newtuple); @@ -420,8 +421,9 @@ ReorderBufferReturnChange(ReorderBuffer *rb, ReorderBufferChange *change) change->data.snapshot = NULL; } break; + /* no data in addition to the struct itself */ + case REORDER_BUFFER_CHANGE_INTERNAL_SPEC_CONFIRM: case REORDER_BUFFER_CHANGE_INTERNAL_COMMAND_ID: - break; case REORDER_BUFFER_CHANGE_INTERNAL_TUPLECID: break; } @@ -1317,6 +1319,7 @@ ReorderBufferCommit(ReorderBuffer *rb, TransactionId xid, PG_TRY(); { ReorderBufferChange *change; + ReorderBufferChange *specinsert = NULL; if (using_subtxn) BeginInternalSubTransaction("replay"); @@ -1333,6 +1336,17 @@ ReorderBufferCommit(ReorderBuffer *rb, TransactionId xid, switch (change->action) { + case REORDER_BUFFER_CHANGE_INTERNAL_SPEC_CONFIRM: + /* + * Confirmation for speculative insertion arrived. Simply + * use as a normal record. It'll be cleaned up at the end + * of INSERT processing. + */ + Assert(specinsert->data.tp.oldtuple == NULL); + change = specinsert; + change->action = REORDER_BUFFER_CHANGE_INSERT; + + /* intentionally fall through */ case REORDER_BUFFER_CHANGE_INSERT: case REORDER_BUFFER_CHANGE_UPDATE: case REORDER_BUFFER_CHANGE_DELETE: @@ -1348,7 +1362,7 @@ ReorderBufferCommit(ReorderBuffer *rb, TransactionId xid, if (reloid == InvalidOid && change->data.tp.newtuple == NULL && change->data.tp.oldtuple == NULL) - continue; + goto change_done; else if (reloid == InvalidOid) elog(ERROR, "could not map filenode \"%s\" to relation OID", relpathperm(change->data.tp.relnode, @@ -1362,50 +1376,92 @@ ReorderBufferCommit(ReorderBuffer *rb, TransactionId xid, relpathperm(change->data.tp.relnode, MAIN_FORKNUM)); - if (RelationIsLogicallyLogged(relation)) + if (!RelationIsLogicallyLogged(relation)) + goto change_done; + + /* + * For now ignore sequence changes entirely. Most of + * the time they don't log changes using records we + * understand, so it doesn't make sense to handle the + * few cases we do. + */ + if (relation->rd_rel->relkind == RELKIND_SEQUENCE) + goto change_done; + + /* user-triggered change */ + if (!IsToastRelation(relation)) { + ReorderBufferToastReplace(rb, txn, relation, change); + rb->apply_change(rb, txn, relation, change); + /* - * For now ignore sequence changes entirely. Most of - * the time they don't log changes using records we - * understand, so it doesn't make sense to handle the - * few cases we do. + * Only clear reassembled toast chunks if we're + * sure they're not required anymore. The creator + * of the tuple tells us. */ - if (relation->rd_rel->relkind == RELKIND_SEQUENCE) - { - } - /* user-triggered change */ - else if (!IsToastRelation(relation)) - { - ReorderBufferToastReplace(rb, txn, relation, change); - rb->apply_change(rb, txn, relation, change); - - /* - * Only clear reassembled toast chunks if we're - * sure they're not required anymore. The creator - * of the tuple tells us. - */ - if (change->data.tp.clear_toast_afterwards) - ReorderBufferToastReset(rb, txn); - } - /* we're not interested in toast deletions */ - else if (change->action == REORDER_BUFFER_CHANGE_INSERT) - { - /* - * Need to reassemble the full toasted Datum in - * memory, to ensure the chunks don't get reused - * till we're done remove it from the list of this - * transaction's changes. Otherwise it will get - * freed/reused while restoring spooled data from - * disk. - */ - dlist_delete(&change->node); - ReorderBufferToastAppendChunk(rb, txn, relation, - change); - } + if (change->data.tp.clear_toast_afterwards) + ReorderBufferToastReset(rb, txn); + } + /* we're not interested in toast deletions */ + else if (change->action == REORDER_BUFFER_CHANGE_INSERT) + { + /* + * Need to reassemble the full toasted Datum in + * memory, to ensure the chunks don't get reused till + * we're done remove it from the list of this + * transaction's changes. Otherwise it will get + * freed/reused while restoring spooled data from + * disk. + */ + dlist_delete(&change->node); + ReorderBufferToastAppendChunk(rb, txn, relation, + change); + } + + change_done: + /* + * Either speculative insertion was confirmed, or it was + * unsuccessful and the record isn't needed anymore. + */ + if (specinsert != NULL) + { + ReorderBufferReturnChange(rb, specinsert); + specinsert = NULL; + } + if (relation != NULL) + { + RelationClose(relation); + relation = NULL; } - RelationClose(relation); break; + + case REORDER_BUFFER_CHANGE_INTERNAL_SPEC_INSERT: + /* + * Speculative insertions are dealt with by delaying the + * processing of the insert until the confirmation record + * arrives. For that we simply unlink the record from the + * chain, so it does not get freed/reused while restoring + * spooled data from disk. + * + * This is safe in the face of concurrent catalog changes + * because the relevant relation can't be changed between + * speculative insertion and confirmation due to + * CheckTableNotInUse() and locking. + */ + + /* clear out a pending (and thus failed) speculation */ + if (specinsert != NULL) + { + ReorderBufferReturnChange(rb, specinsert); + specinsert = NULL; + } + + /* and memorize the pending insertion */ + dlist_delete(&change->node); + specinsert = change; + break; + case REORDER_BUFFER_CHANGE_INTERNAL_SNAPSHOT: /* get rid of the old */ TeardownHistoricSnapshot(false); @@ -1474,6 +1530,17 @@ ReorderBufferCommit(ReorderBuffer *rb, TransactionId xid, } } + /* + * There's a a speculative insertion remaining, just clean in up, it + * can't have been successful, otherwise we'd gotten a confirmation + * record. + */ + if (specinsert) + { + ReorderBufferReturnChange(rb, specinsert); + specinsert = NULL; + } + /* clean up the iterator */ ReorderBufferIterTXNFinish(rb, iterstate); iterstate = NULL; @@ -2001,11 +2068,11 @@ ReorderBufferSerializeChange(ReorderBuffer *rb, ReorderBufferTXN *txn, switch (change->action) { + /* fall through these, they're all similar enough */ case REORDER_BUFFER_CHANGE_INSERT: - /* fall through */ case REORDER_BUFFER_CHANGE_UPDATE: - /* fall through */ case REORDER_BUFFER_CHANGE_DELETE: + case REORDER_BUFFER_CHANGE_INTERNAL_SPEC_INSERT: { char *data; ReorderBufferTupleBuf *oldtup, @@ -2083,9 +2150,8 @@ ReorderBufferSerializeChange(ReorderBuffer *rb, ReorderBufferTXN *txn, } break; } + case REORDER_BUFFER_CHANGE_INTERNAL_SPEC_CONFIRM: case REORDER_BUFFER_CHANGE_INTERNAL_COMMAND_ID: - /* ReorderBufferChange contains everything important */ - break; case REORDER_BUFFER_CHANGE_INTERNAL_TUPLECID: /* ReorderBufferChange contains everything important */ break; @@ -2256,11 +2322,11 @@ ReorderBufferRestoreChange(ReorderBuffer *rb, ReorderBufferTXN *txn, /* restore individual stuff */ switch (change->action) { + /* fall through these, they're all similar enough */ case REORDER_BUFFER_CHANGE_INSERT: - /* fall through */ case REORDER_BUFFER_CHANGE_UPDATE: - /* fall through */ case REORDER_BUFFER_CHANGE_DELETE: + case REORDER_BUFFER_CHANGE_INTERNAL_SPEC_INSERT: if (change->data.tp.newtuple) { Size len = offsetof(ReorderBufferTupleBuf, t_data) + @@ -2309,6 +2375,7 @@ ReorderBufferRestoreChange(ReorderBuffer *rb, ReorderBufferTXN *txn, break; } /* the base struct contains all the data, easy peasy */ + case REORDER_BUFFER_CHANGE_INTERNAL_SPEC_CONFIRM: case REORDER_BUFFER_CHANGE_INTERNAL_COMMAND_ID: case REORDER_BUFFER_CHANGE_INTERNAL_TUPLECID: break; diff --git a/src/backend/rewrite/rewriteHandler.c b/src/backend/rewrite/rewriteHandler.c index 0fc47cb786c..39302a410b8 100644 --- a/src/backend/rewrite/rewriteHandler.c +++ b/src/backend/rewrite/rewriteHandler.c @@ -52,7 +52,10 @@ static Query *rewriteRuleAction(Query *parsetree, CmdType event, bool *returning_flag); static List *adjustJoinTreeList(Query *parsetree, bool removert, int rt_index); -static void rewriteTargetListIU(Query *parsetree, Relation target_relation, +static List *rewriteTargetListIU(List *targetList, + CmdType commandType, + Relation target_relation, + int result_rti, List **attrno_list); static TargetEntry *process_matched_tle(TargetEntry *src_tle, TargetEntry *prior_tle, @@ -66,7 +69,7 @@ static void markQueryForLocking(Query *qry, Node *jtnode, LockClauseStrength strength, LockWaitPolicy waitPolicy, bool pushedDown); static List *matchLocks(CmdType event, RuleLock *rulelocks, - int varno, Query *parsetree); + int varno, Query *parsetree, bool *hasUpdate); static Query *fireRIRrules(Query *parsetree, List *activeRIRs, bool forUpdatePushedDown); static bool view_has_instead_trigger(Relation view, CmdType event); @@ -679,11 +682,13 @@ adjustJoinTreeList(Query *parsetree, bool removert, int rt_index) * order of the original tlist's non-junk entries. This is needed for * processing VALUES RTEs. */ -static void -rewriteTargetListIU(Query *parsetree, Relation target_relation, +static List* +rewriteTargetListIU(List *targetList, + CmdType commandType, + Relation target_relation, + int result_rti, List **attrno_list) { - CmdType commandType = parsetree->commandType; TargetEntry **new_tles; List *new_tlist = NIL; List *junk_tlist = NIL; @@ -709,7 +714,7 @@ rewriteTargetListIU(Query *parsetree, Relation target_relation, new_tles = (TargetEntry **) palloc0(numattrs * sizeof(TargetEntry *)); next_junk_attrno = numattrs + 1; - foreach(temp, parsetree->targetList) + foreach(temp, targetList) { TargetEntry *old_tle = (TargetEntry *) lfirst(temp); @@ -827,7 +832,7 @@ rewriteTargetListIU(Query *parsetree, Relation target_relation, { Node *new_expr; - new_expr = (Node *) makeVar(parsetree->resultRelation, + new_expr = (Node *) makeVar(result_rti, attrno, att_tup->atttypid, att_tup->atttypmod, @@ -846,7 +851,7 @@ rewriteTargetListIU(Query *parsetree, Relation target_relation, pfree(new_tles); - parsetree->targetList = list_concat(new_tlist, junk_tlist); + return list_concat(new_tlist, junk_tlist); } @@ -1288,7 +1293,8 @@ static List * matchLocks(CmdType event, RuleLock *rulelocks, int varno, - Query *parsetree) + Query *parsetree, + bool *hasUpdate) { List *matching_locks = NIL; int nlocks; @@ -1309,6 +1315,9 @@ matchLocks(CmdType event, { RewriteRule *oneLock = rulelocks->rules[i]; + if (oneLock->event == CMD_UPDATE) + *hasUpdate = true; + /* * Suppress ON INSERT/UPDATE/DELETE rules that are disabled or * configured to not fire during the current sessions replication @@ -1766,8 +1775,8 @@ fireRIRrules(Query *parsetree, List *activeRIRs, bool forUpdatePushedDown) /* * Fetch any new security quals that must be applied to this RTE. */ - get_row_security_policies(parsetree, rte, rt_index, - &securityQuals, &withCheckOptions, + get_row_security_policies(parsetree, parsetree->commandType, rte, + rt_index, &securityQuals, &withCheckOptions, &hasRowSecurity, &hasSubLinks); if (securityQuals != NIL || withCheckOptions != NIL) @@ -2642,6 +2651,18 @@ rewriteTargetView(Query *parsetree, Relation view) tle->resno - FirstLowInvalidHeapAttributeNumber); } + if (parsetree->onConflict) + { + foreach(lc, parsetree->onConflict->onConflictSet) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc); + + if (!tle->resjunk) + modified_cols = bms_add_member(modified_cols, + tle->resno - FirstLowInvalidHeapAttributeNumber); + } + } + auto_update_detail = view_cols_are_auto_updatable(viewquery, modified_cols, NULL, @@ -2999,6 +3020,7 @@ RewriteQuery(Query *parsetree, List *rewrite_events) CmdType event = parsetree->commandType; bool instead = false; bool returning = false; + bool updatableview = false; Query *qual_product = NULL; List *rewritten = NIL; ListCell *lc1; @@ -3081,6 +3103,7 @@ RewriteQuery(Query *parsetree, List *rewrite_events) Relation rt_entry_relation; List *locks; List *product_queries; + bool hasUpdate = false; result_relation = parsetree->resultRelation; Assert(result_relation != 0); @@ -3123,19 +3146,41 @@ RewriteQuery(Query *parsetree, List *rewrite_events) List *attrnos; /* Process the main targetlist ... */ - rewriteTargetListIU(parsetree, rt_entry_relation, &attrnos); + parsetree->targetList = rewriteTargetListIU(parsetree->targetList, + parsetree->commandType, + rt_entry_relation, + parsetree->resultRelation, + &attrnos); /* ... and the VALUES expression lists */ rewriteValuesRTE(values_rte, rt_entry_relation, attrnos); } else { /* Process just the main targetlist */ - rewriteTargetListIU(parsetree, rt_entry_relation, NULL); + parsetree->targetList = + rewriteTargetListIU(parsetree->targetList, + parsetree->commandType, + rt_entry_relation, + parsetree->resultRelation, NULL); + } + + if (parsetree->onConflict && + parsetree->onConflict->action == ONCONFLICT_UPDATE) + { + parsetree->onConflict->onConflictSet = + rewriteTargetListIU(parsetree->onConflict->onConflictSet, + CMD_UPDATE, + rt_entry_relation, + parsetree->resultRelation, + NULL); } } else if (event == CMD_UPDATE) { - rewriteTargetListIU(parsetree, rt_entry_relation, NULL); + parsetree->targetList = + rewriteTargetListIU(parsetree->targetList, + parsetree->commandType, rt_entry_relation, + parsetree->resultRelation, NULL); rewriteTargetListUD(parsetree, rt_entry, rt_entry_relation); } else if (event == CMD_DELETE) @@ -3149,7 +3194,7 @@ RewriteQuery(Query *parsetree, List *rewrite_events) * Collect and apply the appropriate rules. */ locks = matchLocks(event, rt_entry_relation->rd_rules, - result_relation, parsetree); + result_relation, parsetree, &hasUpdate); product_queries = fireRules(parsetree, result_relation, @@ -3198,6 +3243,7 @@ RewriteQuery(Query *parsetree, List *rewrite_events) */ instead = true; returning = true; + updatableview = true; } /* @@ -3278,6 +3324,17 @@ RewriteQuery(Query *parsetree, List *rewrite_events) } } + /* + * Updatable views are supported by ON CONFLICT, so don't prevent that + * case from proceeding + */ + if (parsetree->onConflict && + (product_queries != NIL || hasUpdate) && + !updatableview) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("INSERT with ON CONFLICT clause cannot be used with table that has INSERT or UPDATE rules"))); + heap_close(rt_entry_relation, NoLock); } diff --git a/src/backend/rewrite/rowsecurity.c b/src/backend/rewrite/rowsecurity.c index b0b308118f4..2c095ce88ac 100644 --- a/src/backend/rewrite/rowsecurity.c +++ b/src/backend/rewrite/rowsecurity.c @@ -89,9 +89,10 @@ row_security_policy_hook_type row_security_policy_hook_restrictive = NULL; * set to true if any of the quals returned contain sublinks. */ void -get_row_security_policies(Query* root, RangeTblEntry* rte, int rt_index, - List **securityQuals, List **withCheckOptions, - bool *hasRowSecurity, bool *hasSubLinks) +get_row_security_policies(Query* root, CmdType commandType, RangeTblEntry* rte, + int rt_index, List **securityQuals, + List **withCheckOptions, bool *hasRowSecurity, + bool *hasSubLinks) { Expr *rowsec_expr = NULL; Expr *rowsec_with_check_expr = NULL; @@ -159,7 +160,7 @@ get_row_security_policies(Query* root, RangeTblEntry* rte, int rt_index, /* Grab the built-in policies which should be applied to this relation. */ rel = heap_open(rte->relid, NoLock); - rowsec_policies = pull_row_security_policies(root->commandType, rel, + rowsec_policies = pull_row_security_policies(commandType, rel, user_id); /* @@ -201,7 +202,7 @@ get_row_security_policies(Query* root, RangeTblEntry* rte, int rt_index, */ if (row_security_policy_hook_restrictive) { - hook_policies_restrictive = (*row_security_policy_hook_restrictive)(root->commandType, rel); + hook_policies_restrictive = (*row_security_policy_hook_restrictive)(commandType, rel); /* Build the expression from any policies returned. */ if (hook_policies_restrictive != NIL) @@ -214,7 +215,7 @@ get_row_security_policies(Query* root, RangeTblEntry* rte, int rt_index, if (row_security_policy_hook_permissive) { - hook_policies_permissive = (*row_security_policy_hook_permissive)(root->commandType, rel); + hook_policies_permissive = (*row_security_policy_hook_permissive)(commandType, rel); /* Build the expression from any policies returned. */ if (hook_policies_permissive != NIL) @@ -242,7 +243,7 @@ get_row_security_policies(Query* root, RangeTblEntry* rte, int rt_index, * WITH CHECK policy (this will be a copy of the USING policy, if no * explicit WITH CHECK policy exists). */ - if (root->commandType == CMD_INSERT || root->commandType == CMD_UPDATE) + if (commandType == CMD_INSERT || commandType == CMD_UPDATE) { /* * WITH CHECK OPTIONS wants a WCO node which wraps each Expr, so @@ -259,7 +260,7 @@ get_row_security_policies(Query* root, RangeTblEntry* rte, int rt_index, WithCheckOption *wco; wco = (WithCheckOption *) makeNode(WithCheckOption); - wco->kind = root->commandType == CMD_INSERT ? WCO_RLS_INSERT_CHECK : + wco->kind = commandType == CMD_INSERT ? WCO_RLS_INSERT_CHECK : WCO_RLS_UPDATE_CHECK; wco->relname = pstrdup(RelationGetRelationName(rel)); wco->qual = (Node *) hook_with_check_expr_restrictive; @@ -276,7 +277,7 @@ get_row_security_policies(Query* root, RangeTblEntry* rte, int rt_index, WithCheckOption *wco; wco = (WithCheckOption *) makeNode(WithCheckOption); - wco->kind = root->commandType == CMD_INSERT ? WCO_RLS_INSERT_CHECK : + wco->kind = commandType == CMD_INSERT ? WCO_RLS_INSERT_CHECK : WCO_RLS_UPDATE_CHECK; wco->relname = pstrdup(RelationGetRelationName(rel)); wco->qual = (Node *) rowsec_with_check_expr; @@ -289,7 +290,7 @@ get_row_security_policies(Query* root, RangeTblEntry* rte, int rt_index, WithCheckOption *wco; wco = (WithCheckOption *) makeNode(WithCheckOption); - wco->kind = root->commandType == CMD_INSERT ? WCO_RLS_INSERT_CHECK : + wco->kind = commandType == CMD_INSERT ? WCO_RLS_INSERT_CHECK : WCO_RLS_UPDATE_CHECK; wco->relname = pstrdup(RelationGetRelationName(rel)); wco->qual = (Node *) hook_with_check_expr_permissive; @@ -312,19 +313,72 @@ get_row_security_policies(Query* root, RangeTblEntry* rte, int rt_index, combined_qual_eval = makeBoolExpr(OR_EXPR, combined_quals, -1); wco = (WithCheckOption *) makeNode(WithCheckOption); - wco->kind = root->commandType == CMD_INSERT ? WCO_RLS_INSERT_CHECK : + wco->kind = commandType == CMD_INSERT ? WCO_RLS_INSERT_CHECK : WCO_RLS_UPDATE_CHECK; wco->relname = pstrdup(RelationGetRelationName(rel)); wco->qual = (Node *) combined_qual_eval; wco->cascaded = false; *withCheckOptions = lappend(*withCheckOptions, wco); } + + /* + * ON CONFLICT DO UPDATE has an RTE that is subject to both INSERT and + * UPDATE RLS enforcement. Those are enforced (as a special, distinct + * kind of WCO) on the target tuple. + * + * Make a second, recursive pass over the RTE for this, gathering + * UPDATE-applicable RLS checks/WCOs, and gathering and converting + * UPDATE-applicable security quals into WCO_RLS_CONFLICT_CHECK RLS + * checks/WCOs. Finally, these distinct kinds of RLS checks/WCOs are + * concatenated with our own INSERT-applicable list. + */ + if (root->onConflict && root->onConflict->action == ONCONFLICT_UPDATE && + commandType == CMD_INSERT) + { + List *conflictSecurityQuals = NIL; + List *conflictWCOs = NIL; + ListCell *item; + bool conflictHasRowSecurity = false; + bool conflictHasSublinks = false; + + /* Assume that RTE is target resultRelation */ + get_row_security_policies(root, CMD_UPDATE, rte, rt_index, + &conflictSecurityQuals, &conflictWCOs, + &conflictHasRowSecurity, + &conflictHasSublinks); + + if (conflictHasRowSecurity) + *hasRowSecurity = true; + if (conflictHasSublinks) + *hasSubLinks = true; + + /* + * Append WITH CHECK OPTIONs/RLS checks, which should not conflict + * between this INSERT and the auxiliary UPDATE + */ + *withCheckOptions = list_concat(*withCheckOptions, + conflictWCOs); + + foreach(item, conflictSecurityQuals) + { + Expr *conflict_rowsec_expr = (Expr *) lfirst(item); + WithCheckOption *wco; + + wco = (WithCheckOption *) makeNode(WithCheckOption); + + wco->kind = WCO_RLS_CONFLICT_CHECK; + wco->relname = pstrdup(RelationGetRelationName(rel)); + wco->qual = (Node *) copyObject(conflict_rowsec_expr); + wco->cascaded = false; + *withCheckOptions = lappend(*withCheckOptions, wco); + } + } } /* For SELECT, UPDATE, and DELETE, set the security quals */ - if (root->commandType == CMD_SELECT - || root->commandType == CMD_UPDATE - || root->commandType == CMD_DELETE) + if (commandType == CMD_SELECT + || commandType == CMD_UPDATE + || commandType == CMD_DELETE) { /* restrictive policies can simply be added to the list first */ if (hook_expr_restrictive) diff --git a/src/backend/storage/lmgr/lmgr.c b/src/backend/storage/lmgr/lmgr.c index d13a1673344..c0529497496 100644 --- a/src/backend/storage/lmgr/lmgr.c +++ b/src/backend/storage/lmgr/lmgr.c @@ -26,6 +26,24 @@ /* + * Per-backend counter for generating speculative insertion tokens. + * + * This may wrap around, but that's OK as it's only used for the short + * duration between inserting a tuple and checking that there are no (unique) + * constraint violations. It's theoretically possible that a backend sees a + * tuple that was speculatively inserted by another backend, but before it has + * started waiting on the token, the other backend completes its insertion, + * and then then performs 2^32 unrelated insertions. And after all that, the + * first backend finally calls SpeculativeInsertionLockAcquire(), with the + * intention of waiting for the first insertion to complete, but ends up + * waiting for the latest unrelated insertion instead. Even then, nothing + * particularly bad happens: in the worst case they deadlock, causing one of + * the transactions to abort. + */ +static uint32 speculativeInsertionToken = 0; + + +/* * Struct to hold context info for transaction lock waits. * * 'oper' is the operation that needs to wait for the other transaction; 'rel' @@ -576,6 +594,73 @@ ConditionalXactLockTableWait(TransactionId xid) } /* + * SpeculativeInsertionLockAcquire + * + * Insert a lock showing that the given transaction ID is inserting a tuple, + * but hasn't yet decided whether it's going to keep it. The lock can then be + * used to wait for the decision to go ahead with the insertion, or aborting + * it. + * + * The token is used to distinguish multiple insertions by the same + * transaction. It is returned to caller. + */ +uint32 +SpeculativeInsertionLockAcquire(TransactionId xid) +{ + LOCKTAG tag; + + speculativeInsertionToken++; + + /* + * Check for wrap-around. Zero means no token is held, so don't use that. + */ + if (speculativeInsertionToken == 0) + speculativeInsertionToken = 1; + + SET_LOCKTAG_SPECULATIVE_INSERTION(tag, xid, speculativeInsertionToken); + + (void) LockAcquire(&tag, ExclusiveLock, false, false); + + return speculativeInsertionToken; +} + +/* + * SpeculativeInsertionLockRelease + * + * Delete the lock showing that the given transaction is speculatively + * inserting a tuple. + */ +void +SpeculativeInsertionLockRelease(TransactionId xid) +{ + LOCKTAG tag; + + SET_LOCKTAG_SPECULATIVE_INSERTION(tag, xid, speculativeInsertionToken); + + LockRelease(&tag, ExclusiveLock, false); +} + +/* + * SpeculativeInsertionWait + * + * Wait for the specified transaction to finish or abort the insertion of a + * tuple. + */ +void +SpeculativeInsertionWait(TransactionId xid, uint32 token) +{ + LOCKTAG tag; + + SET_LOCKTAG_SPECULATIVE_INSERTION(tag, xid, token); + + Assert(TransactionIdIsValid(xid)); + Assert(token != 0); + + (void) LockAcquire(&tag, ShareLock, false, false); + LockRelease(&tag, ShareLock, false); +} + +/* * XactLockTableWaitErrorContextCb * Error context callback for transaction lock waits. */ @@ -873,6 +958,12 @@ DescribeLockTag(StringInfo buf, const LOCKTAG *tag) tag->locktag_field1, tag->locktag_field2); break; + case LOCKTAG_SPECULATIVE_TOKEN: + appendStringInfo(buf, + _("speculative token %u of transaction %u"), + tag->locktag_field2, + tag->locktag_field1); + break; case LOCKTAG_OBJECT: appendStringInfo(buf, _("object %u of class %u of database %u"), diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c index 9c14e8abdf8..bcffd85754c 100644 --- a/src/backend/tcop/pquery.c +++ b/src/backend/tcop/pquery.c @@ -202,8 +202,14 @@ ProcessQuery(PlannedStmt *plan, lastOid = queryDesc->estate->es_lastoid; else lastOid = InvalidOid; - snprintf(completionTag, COMPLETION_TAG_BUFSIZE, - "INSERT %u %u", lastOid, queryDesc->estate->es_processed); + if (plan->isUpsert) + snprintf(completionTag, COMPLETION_TAG_BUFSIZE, + "UPSERT %u %u", + lastOid, queryDesc->estate->es_processed); + else + snprintf(completionTag, COMPLETION_TAG_BUFSIZE, + "INSERT %u %u", + lastOid, queryDesc->estate->es_processed); break; case CMD_UPDATE: snprintf(completionTag, COMPLETION_TAG_BUFSIZE, @@ -1356,7 +1362,10 @@ PortalRunMulti(Portal portal, bool isTopLevel, * 0" here because technically there is no query of the matching tag type, * and printing a non-zero count for a different query type seems wrong, * e.g. an INSERT that does an UPDATE instead should not print "0 1" if - * one row was updated. See QueryRewrite(), step 3, for details. + * one row was updated (unless the ON CONFLICT DO UPDATE, or "UPSERT" + * variant of INSERT was used to update the row, where it's logically a + * direct effect of the top level command). See QueryRewrite(), step 3, + * for details. */ if (completionTag && completionTag[0] == '\0') { @@ -1366,6 +1375,8 @@ PortalRunMulti(Portal portal, bool isTopLevel, sprintf(completionTag, "SELECT 0 0"); else if (strcmp(completionTag, "INSERT") == 0) strcpy(completionTag, "INSERT 0 0"); + else if (strcmp(completionTag, "UPSERT") == 0) + strcpy(completionTag, "UPSERT 0 0"); else if (strcmp(completionTag, "UPDATE") == 0) strcpy(completionTag, "UPDATE 0"); else if (strcmp(completionTag, "DELETE") == 0) diff --git a/src/backend/utils/adt/lockfuncs.c b/src/backend/utils/adt/lockfuncs.c index 491824dd6bf..9d53a8b6a32 100644 --- a/src/backend/utils/adt/lockfuncs.c +++ b/src/backend/utils/adt/lockfuncs.c @@ -29,6 +29,7 @@ static const char *const LockTagTypeNames[] = { "tuple", "transactionid", "virtualxid", + "speculative token", "object", "userlock", "advisory" diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 69267bdb918..4b3cd85ad90 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -354,6 +354,9 @@ static void get_select_query_def(Query *query, deparse_context *context, TupleDesc resultDesc); static void get_insert_query_def(Query *query, deparse_context *context); static void get_update_query_def(Query *query, deparse_context *context); +static void get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, + RangeTblEntry *rte); static void get_delete_query_def(Query *query, deparse_context *context); static void get_utility_query_def(Query *query, deparse_context *context); static void get_basic_select_query(Query *query, deparse_context *context, @@ -3846,15 +3849,23 @@ set_deparse_planstate(deparse_namespace *dpns, PlanState *ps) * For a SubqueryScan, pretend the subplan is INNER referent. (We don't * use OUTER because that could someday conflict with the normal meaning.) * Likewise, for a CteScan, pretend the subquery's plan is INNER referent. + * For ON CONFLICT .. UPDATE we just need the inner tlist to point to the + * excluded expression's tlist. (Similar to the SubqueryScan we don't want + * to reuse OUTER, it's used for RETURNING in some modify table cases, + * although not INSERT .. CONFLICT). */ if (IsA(ps, SubqueryScanState)) dpns->inner_planstate = ((SubqueryScanState *) ps)->subplan; else if (IsA(ps, CteScanState)) dpns->inner_planstate = ((CteScanState *) ps)->cteplanstate; + else if (IsA(ps, ModifyTableState)) + dpns->inner_planstate = ps; else dpns->inner_planstate = innerPlanState(ps); - if (dpns->inner_planstate) + if (IsA(ps, ModifyTableState)) + dpns->inner_tlist = ((ModifyTableState *) ps)->mt_excludedtlist; + else if (dpns->inner_planstate) dpns->inner_tlist = dpns->inner_planstate->plan->targetlist; else dpns->inner_tlist = NIL; @@ -5302,6 +5313,32 @@ get_insert_query_def(Query *query, deparse_context *context) appendStringInfoString(buf, "DEFAULT VALUES"); } + /* Add ON CONFLICT if present */ + if (query->onConflict) + { + OnConflictExpr *confl = query->onConflict; + + if (confl->action == ONCONFLICT_NOTHING) + { + appendStringInfoString(buf, " ON CONFLICT DO NOTHING"); + } + else + { + appendStringInfoString(buf, " ON CONFLICT DO UPDATE SET "); + /* Deparse targetlist */ + get_update_query_targetlist_def(query, confl->onConflictSet, + context, rte); + + /* Add a WHERE clause if given */ + if (confl->onConflictWhere != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->onConflictWhere, context, false); + } + } + } + /* Add RETURNING if present */ if (query->returningList) { @@ -5321,12 +5358,6 @@ get_update_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; - List *ma_sublinks; - ListCell *next_ma_cell; - SubLink *cur_ma_sublink; - int remaining_ma_columns; - const char *sep; - ListCell *l; /* Insert the WITH clause if given */ get_with_clause(query, context); @@ -5349,6 +5380,46 @@ get_update_query_def(Query *query, deparse_context *context) quote_identifier(rte->alias->aliasname)); appendStringInfoString(buf, " SET "); + /* Deparse targetlist */ + get_update_query_targetlist_def(query, query->targetList, context, rte); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_update_query_targetlist_def - Parse back an UPDATE targetlist + * ---------- + */ +static void +get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, RangeTblEntry *rte) +{ + StringInfo buf = context->buf; + ListCell *l; + ListCell *next_ma_cell; + int remaining_ma_columns; + const char *sep; + SubLink *cur_ma_sublink; + List *ma_sublinks; + /* * Prepare to deal with MULTIEXPR assignments: collect the source SubLinks * into a list. We expect them to appear, in ID order, in resjunk tlist @@ -5357,7 +5428,7 @@ get_update_query_def(Query *query, deparse_context *context) ma_sublinks = NIL; if (query->hasSubLinks) /* else there can't be any */ { - foreach(l, query->targetList) + foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); @@ -5379,7 +5450,7 @@ get_update_query_def(Query *query, deparse_context *context) /* Add the comma separated list of 'attname = value' */ sep = ""; - foreach(l, query->targetList) + foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); Node *expr; @@ -5470,25 +5541,6 @@ get_update_query_def(Query *query, deparse_context *context) get_rule_expr(expr, context, false); } - - /* Add the FROM clause if needed */ - get_from_clause(query, " FROM ", context); - - /* Add a WHERE clause if given */ - if (query->jointree->quals != NULL) - { - appendContextKeyword(context, " WHERE ", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_rule_expr(query->jointree->quals, context, false); - } - - /* Add RETURNING if present */ - if (query->returningList) - { - appendContextKeyword(context, " RETURNING", - -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); - get_target_list(query->returningList, context, NULL); - } } diff --git a/src/backend/utils/time/tqual.c b/src/backend/utils/time/tqual.c index a4a478d1142..b4284d6d94f 100644 --- a/src/backend/utils/time/tqual.c +++ b/src/backend/utils/time/tqual.c @@ -405,6 +405,13 @@ HeapTupleSatisfiesToast(HeapTuple htup, Snapshot snapshot, } } } + /* + * An invalid Xmin can be left behind by a speculative insertion that + * is cancelled by super-deleting the tuple. We shouldn't see any of + * those in TOAST tables, but better safe than sorry. + */ + else if (!TransactionIdIsValid(HeapTupleHeaderGetXmin(tuple))) + return false; } /* otherwise assume the tuple is valid for TOAST. */ @@ -714,8 +721,11 @@ HeapTupleSatisfiesUpdate(HeapTuple htup, CommandId curcid, * output argument to return the xids of concurrent xacts that affected the * tuple. snapshot->xmin is set to the tuple's xmin if that is another * transaction that's still in progress; or to InvalidTransactionId if the - * tuple's xmin is committed good, committed dead, or my own xact. Similarly - * for snapshot->xmax and the tuple's xmax. + * tuple's xmin is committed good, committed dead, or my own xact. + * Similarly for snapshot->xmax and the tuple's xmax. If the tuple was + * inserted speculatively, meaning that the inserter might still back down + * on the insertion without aborting the whole transaction, the associated + * token is also returned in snapshot->speculativeToken. */ bool HeapTupleSatisfiesDirty(HeapTuple htup, Snapshot snapshot, @@ -727,6 +737,7 @@ HeapTupleSatisfiesDirty(HeapTuple htup, Snapshot snapshot, Assert(htup->t_tableOid != InvalidOid); snapshot->xmin = snapshot->xmax = InvalidTransactionId; + snapshot->speculativeToken = 0; if (!HeapTupleHeaderXminCommitted(tuple)) { @@ -808,6 +819,20 @@ HeapTupleSatisfiesDirty(HeapTuple htup, Snapshot snapshot, } else if (TransactionIdIsInProgress(HeapTupleHeaderGetRawXmin(tuple))) { + /* + * Return the speculative token to caller. Caller can worry + * about xmax, since it requires a conclusively locked row + * version, and a concurrent update to this tuple is a conflict + * of its purposes. + */ + if (HeapTupleHeaderIsSpeculative(tuple)) + { + snapshot->speculativeToken = + HeapTupleHeaderGetSpeculativeToken(tuple); + + Assert(snapshot->speculativeToken != 0); + } + snapshot->xmin = HeapTupleHeaderGetRawXmin(tuple); /* XXX shouldn't we fall through to look at xmax? */ return true; /* in insertion by other */ diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c index ff01368531a..f4155f78770 100644 --- a/src/bin/psql/common.c +++ b/src/bin/psql/common.c @@ -894,9 +894,12 @@ PrintQueryResults(PGresult *results) success = StoreQueryTuple(results); else success = PrintQueryTuples(results); - /* if it's INSERT/UPDATE/DELETE RETURNING, also print status */ + /* + * if it's INSERT/UPSERT/UPDATE/DELETE RETURNING, also print status + */ cmdstatus = PQcmdStatus(results); if (strncmp(cmdstatus, "INSERT", 6) == 0 || + strncmp(cmdstatus, "UPSERT", 6) == 0 || strncmp(cmdstatus, "UPDATE", 6) == 0 || strncmp(cmdstatus, "DELETE", 6) == 0) PrintQueryStatus(results); diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 888cce7a2d8..49c8ca4d66e 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -28,6 +28,7 @@ #define HEAP_INSERT_SKIP_WAL 0x0001 #define HEAP_INSERT_SKIP_FSM 0x0002 #define HEAP_INSERT_FROZEN 0x0004 +#define HEAP_INSERT_SPECULATIVE 0x0008 typedef struct BulkInsertStateData *BulkInsertState; @@ -142,6 +143,8 @@ extern void heap_multi_insert(Relation relation, HeapTuple *tuples, int ntuples, extern HTSU_Result heap_delete(Relation relation, ItemPointer tid, CommandId cid, Snapshot crosscheck, bool wait, HeapUpdateFailureData *hufd); +extern void heap_finish_speculative(Relation relation, HeapTuple tuple); +extern void heap_abort_speculative(Relation relation, HeapTuple tuple); extern HTSU_Result heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, CommandId cid, Snapshot crosscheck, bool wait, diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h index f0f89dec0f2..caa0f14f4bf 100644 --- a/src/include/access/heapam_xlog.h +++ b/src/include/access/heapam_xlog.h @@ -34,7 +34,7 @@ #define XLOG_HEAP_UPDATE 0x20 /* 0x030 is free, was XLOG_HEAP_MOVE */ #define XLOG_HEAP_HOT_UPDATE 0x40 -/* 0x050 is free, was XLOG_HEAP_NEWPAGE */ +#define XLOG_HEAP_CONFIRM 0x50 #define XLOG_HEAP_LOCK 0x60 #define XLOG_HEAP_INPLACE 0x70 @@ -60,23 +60,43 @@ #define XLOG_HEAP2_NEW_CID 0x70 /* - * xl_heap_* ->flag values, 8 bits are available. + * xl_heap_insert/xl_heap_multi_insert flag values, 8 bits are available. */ /* PD_ALL_VISIBLE was cleared */ -#define XLOG_HEAP_ALL_VISIBLE_CLEARED (1<<0) +#define XLH_INSERT_ALL_VISIBLE_CLEARED (1<<0) +#define XLH_INSERT_LAST_IN_MULTI (1<<1) +#define XLH_INSERT_IS_SPECULATIVE (1<<2) +#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3) + +/* + * xl_heap_update flag values, 8 bits are available. + */ +/* PD_ALL_VISIBLE was cleared */ +#define XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED (1<<0) /* PD_ALL_VISIBLE was cleared in the 2nd page */ -#define XLOG_HEAP_NEW_ALL_VISIBLE_CLEARED (1<<1) -#define XLOG_HEAP_CONTAINS_OLD_TUPLE (1<<2) -#define XLOG_HEAP_CONTAINS_OLD_KEY (1<<3) -#define XLOG_HEAP_CONTAINS_NEW_TUPLE (1<<4) -#define XLOG_HEAP_PREFIX_FROM_OLD (1<<5) -#define XLOG_HEAP_SUFFIX_FROM_OLD (1<<6) -/* last xl_heap_multi_insert record for one heap_multi_insert() call */ -#define XLOG_HEAP_LAST_MULTI_INSERT (1<<7) +#define XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED (1<<1) +#define XLH_UPDATE_CONTAINS_OLD_TUPLE (1<<2) +#define XLH_UPDATE_CONTAINS_OLD_KEY (1<<3) +#define XLH_UPDATE_CONTAINS_NEW_TUPLE (1<<4) +#define XLH_UPDATE_PREFIX_FROM_OLD (1<<5) +#define XLH_UPDATE_SUFFIX_FROM_OLD (1<<6) /* convenience macro for checking whether any form of old tuple was logged */ -#define XLOG_HEAP_CONTAINS_OLD \ - (XLOG_HEAP_CONTAINS_OLD_TUPLE | XLOG_HEAP_CONTAINS_OLD_KEY) +#define XLH_UPDATE_CONTAINS_OLD \ + (XLH_UPDATE_CONTAINS_OLD_TUPLE | XLH_UPDATE_CONTAINS_OLD_KEY) + +/* + * xl_heap_delete flag values, 8 bits are available. + */ +/* PD_ALL_VISIBLE was cleared */ +#define XLH_DELETE_ALL_VISIBLE_CLEARED (1<<0) +#define XLH_DELETE_CONTAINS_OLD_TUPLE (1<<1) +#define XLH_DELETE_CONTAINS_OLD_KEY (1<<2) +#define XLH_DELETE_IS_SUPER (1<<3) + +/* convenience macro for checking whether any form of old tuple was logged */ +#define XLH_DELETE_CONTAINS_OLD \ + (XLH_DELETE_CONTAINS_OLD_TUPLE | XLH_DELETE_CONTAINS_OLD_KEY) /* This is what we need to know about delete */ typedef struct xl_heap_delete @@ -243,6 +263,14 @@ typedef struct xl_heap_lock_updated #define SizeOfHeapLockUpdated (offsetof(xl_heap_lock_updated, infobits_set) + sizeof(uint8)) +/* This is what we need to know about confirmation of speculative insertion */ +typedef struct xl_heap_confirm +{ + OffsetNumber offnum; /* confirmed tuple's offset on page */ +} xl_heap_confirm; + +#define SizeOfHeapConfirm (offsetof(xl_heap_confirm, offnum) + sizeof(OffsetNumber)) + /* This is what we need to know about in-place update */ typedef struct xl_heap_inplace { diff --git a/src/include/access/hio.h b/src/include/access/hio.h index b0140298b1f..b3b91e70d50 100644 --- a/src/include/access/hio.h +++ b/src/include/access/hio.h @@ -36,7 +36,7 @@ typedef struct BulkInsertStateData extern void RelationPutHeapTuple(Relation relation, Buffer buffer, - HeapTuple tuple); + HeapTuple tuple, bool token); extern Buffer RelationGetBufferForTuple(Relation relation, Size len, Buffer otherBuffer, int options, BulkInsertState bistate, diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h index 0a673cd5267..80285acc3b6 100644 --- a/src/include/access/htup_details.h +++ b/src/include/access/htup_details.h @@ -96,6 +96,15 @@ * unrelated tuple stored into a slot recently freed by VACUUM. If either * check fails, one may assume that there is no live descendant version. * + * t_ctid is sometimes used to store a speculative insertion token, instead + * of a real TID. A speculative token is set on a tuple that's being + * inserted, until the inserter is sure that it wants to go ahead with the + * insertion. Hence a token should only be seen on a tuple with an XMAX + * that's still in-progress, or invalid/aborted. The token is replaced with + * the tuple's real TID when the insertion is confirmed. One should never + * see a speculative insertion token while following a chain of t_ctid links, + * because they are not used on updates, only insertions. + * * Following the fixed header fields, the nulls bitmap is stored (beginning * at t_bits). The bitmap is *not* stored if t_infomask shows that there * are no nulls in the tuple. If an OID field is present (as indicated by @@ -138,7 +147,8 @@ struct HeapTupleHeaderData DatumTupleFields t_datum; } t_choice; - ItemPointerData t_ctid; /* current TID of this or newer tuple */ + ItemPointerData t_ctid; /* current TID of this or newer tuple (or a + * speculative insertion token) */ /* Fields below here must match MinimalTupleData! */ @@ -242,6 +252,14 @@ struct HeapTupleHeaderData #define HEAP_TUPLE_HAS_MATCH HEAP_ONLY_TUPLE /* tuple has a join match */ /* + * Special value used in t_ctid.ip_posid, to indicate that it holds a + * speculative insertion token rather than a real TID. This must be higher + * than MaxOffsetNumber, so that it can be distinguished from a valid + * offset number in a regular item pointer. + */ +#define SpecTokenOffsetNumber 0xfffe + +/* * HeapTupleHeader accessor macros * * Note: beware of multiple evaluations of "tup" argument. But the Set @@ -377,6 +395,22 @@ do { \ (tup)->t_choice.t_heap.t_field3.t_xvac = (xid); \ } while (0) +#define HeapTupleHeaderIsSpeculative(tup) \ +( \ + (tup)->t_ctid.ip_posid == SpecTokenOffsetNumber \ +) + +#define HeapTupleHeaderGetSpeculativeToken(tup) \ +( \ + AssertMacro(HeapTupleHeaderIsSpeculative(tup)), \ + ItemPointerGetBlockNumber(&(tup)->t_ctid) \ +) + +#define HeapTupleHeaderSetSpeculativeToken(tup, token) \ +( \ + ItemPointerSet(&(tup)->t_ctid, token, SpecTokenOffsetNumber) \ +) + #define HeapTupleHeaderGetDatumLength(tup) \ VARSIZE(tup) diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 3d2c5b2a293..969a8e3c358 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201505081 +#define CATALOG_VERSION_NO 201505082 #endif diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h index a04def96e4c..06f38202a5f 100644 --- a/src/include/catalog/index.h +++ b/src/include/catalog/index.h @@ -81,6 +81,8 @@ extern void index_drop(Oid indexId, bool concurrent); extern IndexInfo *BuildIndexInfo(Relation index); +extern void BuildSpeculativeIndexInfo(Relation index, IndexInfo *ii); + extern void FormIndexDatum(IndexInfo *indexInfo, TupleTableSlot *slot, EState *estate, diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index 33c8fad844c..1b68b54c7dd 100644 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -195,6 +195,7 @@ extern void ExecConstraints(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate); extern void ExecWithCheckOptions(WCOKind kind, ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate); +extern LockTupleMode ExecUpdateLockMode(EState *estate, ResultRelInfo *relinfo); extern ExecRowMark *ExecFindRowMark(EState *estate, Index rti); extern ExecAuxRowMark *ExecBuildAuxRowMark(ExecRowMark *erm, List *targetlist); extern TupleTableSlot *EvalPlanQual(EState *estate, EPQState *epqstate, @@ -361,16 +362,18 @@ extern void UnregisterExprContextCallback(ExprContext *econtext, /* * prototypes from functions in execIndexing.c */ -extern void ExecOpenIndices(ResultRelInfo *resultRelInfo); +extern void ExecOpenIndices(ResultRelInfo *resultRelInfo, bool speculative); extern void ExecCloseIndices(ResultRelInfo *resultRelInfo); extern List *ExecInsertIndexTuples(TupleTableSlot *slot, ItemPointer tupleid, - EState *estate); -extern bool check_exclusion_constraint(Relation heap, Relation index, + EState *estate, bool noDupErr, bool *specConflict, + List *arbiterIndexes); +extern bool ExecCheckIndexConstraints(TupleTableSlot *slot, EState *estate, + ItemPointer conflictTid, List *arbiterIndexes); +extern void check_exclusion_constraint(Relation heap, Relation index, IndexInfo *indexInfo, ItemPointer tupleid, Datum *values, bool *isnull, - EState *estate, - bool newIndex, bool errorOK); + EState *estate, bool newIndex); #endif /* EXECUTOR_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index dac542fbc14..210dbe4a534 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -41,6 +41,9 @@ * ExclusionOps Per-column exclusion operators, or NULL if none * ExclusionProcs Underlying function OIDs for ExclusionOps * ExclusionStrats Opclass strategy numbers for ExclusionOps + * UniqueOps Theses are like Exclusion*, but for unique indexes + * UniqueProcs + * UniqueStrats * Unique is it a unique index? * ReadyForInserts is it valid for inserts? * Concurrent are we doing a concurrent index build? @@ -62,6 +65,9 @@ typedef struct IndexInfo Oid *ii_ExclusionOps; /* array with one entry per column */ Oid *ii_ExclusionProcs; /* array with one entry per column */ uint16 *ii_ExclusionStrats; /* array with one entry per column */ + Oid *ii_UniqueOps; /* array with one entry per column */ + Oid *ii_UniqueProcs; /* array with one entry per column */ + uint16 *ii_UniqueStrats; /* array with one entry per column */ bool ii_Unique; bool ii_ReadyForInserts; bool ii_Concurrent; @@ -308,6 +314,8 @@ typedef struct JunkFilter * ConstraintExprs array of constraint-checking expr states * junkFilter for removing junk attributes from tuples * projectReturning for computing a RETURNING list + * onConflictSetProj for computing ON CONFLICT DO UPDATE SET + * onConflictSetWhere list of ON CONFLICT DO UPDATE exprs (qual) * ---------------- */ typedef struct ResultRelInfo @@ -329,6 +337,8 @@ typedef struct ResultRelInfo List **ri_ConstraintExprs; JunkFilter *ri_junkFilter; ProjectionInfo *ri_projectReturning; + ProjectionInfo *ri_onConflictSetProj; + List *ri_onConflictSetWhere; } ResultRelInfo; /* ---------------- @@ -1094,6 +1104,11 @@ typedef struct ModifyTableState List **mt_arowmarks; /* per-subplan ExecAuxRowMark lists */ EPQState mt_epqstate; /* for evaluating EvalPlanQual rechecks */ bool fireBSTriggers; /* do we need to fire stmt triggers? */ + OnConflictAction mt_onconflict; /* ON CONFLICT type */ + List *mt_arbiterindexes; /* unique index OIDs to arbitrate taking alt path */ + TupleTableSlot *mt_existing; /* slot to store existing target tuple in */ + List *mt_excludedtlist; /* the excluded pseudo relation's tlist */ + TupleTableSlot *mt_conflproj; /* FIXME*/ } ModifyTableState; /* ---------------- diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 8991f3f80ca..768f413a451 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -168,10 +168,12 @@ typedef enum NodeTag T_CoerceToDomainValue, T_SetToDefault, T_CurrentOfExpr, + T_InferenceElem, T_TargetEntry, T_RangeTblRef, T_JoinExpr, T_FromExpr, + T_OnConflictExpr, T_IntoClause, /* @@ -413,6 +415,8 @@ typedef enum NodeTag T_RowMarkClause, T_XmlSerialize, T_WithClause, + T_InferClause, + T_OnConflictClause, T_CommonTableExpr, T_RoleSpec, @@ -626,4 +630,17 @@ typedef enum JoinType (1 << JOIN_RIGHT) | \ (1 << JOIN_ANTI))) != 0) +/* + * OnConflictAction - + * "ON CONFLICT" clause type of query + * + * This is needed in both parsenodes.h and plannodes.h, so put it here... + */ +typedef enum OnConflictAction +{ + ONCONFLICT_NONE, /* No "ON CONFLICT" clause */ + ONCONFLICT_NOTHING, /* ON CONFLICT ... DO NOTHING */ + ONCONFLICT_UPDATE /* ON CONFLICT ... DO UPDATE */ +} OnConflictAction; + #endif /* NODES_H */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 7d15ef2847b..91ca9c6fd0e 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -132,6 +132,8 @@ typedef struct Query List *withCheckOptions; /* a list of WithCheckOption's */ + OnConflictExpr *onConflict; /* ON CONFLICT DO [NOTHING | UPDATE] */ + List *returningList; /* return-values list (of TargetEntry) */ List *groupClause; /* a list of SortGroupClause's */ @@ -591,7 +593,7 @@ typedef enum TableLikeOption } TableLikeOption; /* - * IndexElem - index parameters (used in CREATE INDEX) + * IndexElem - index parameters (used in CREATE INDEX, and in ON CONFLICT) * * For a plain index attribute, 'name' is the name of the table column to * index, and 'expr' is NULL. For an index expression, 'name' is NULL and @@ -735,9 +737,9 @@ typedef struct XmlSerialize * For SELECT/INSERT/UPDATE permissions, if the user doesn't have * table-wide permissions then it is sufficient to have the permissions * on all columns identified in selectedCols (for SELECT) and/or - * insertedCols and/or updatedCols (INSERT with ON CONFLICT UPDATE may - * have all 3). selectedCols, insertedCols and updatedCols are - * bitmapsets, which cannot have negative integer members, so we subtract + * insertedCols and/or updatedCols (INSERT with ON CONFLICT DO UPDATE may + * have all 3). selectedCols, insertedCols and updatedCols are bitmapsets, + * which cannot have negative integer members, so we subtract * FirstLowInvalidHeapAttributeNumber from column numbers before storing * them in these fields. A whole-row Var reference is represented by * setting the bit for InvalidAttrNumber. @@ -881,7 +883,8 @@ typedef enum WCOKind { WCO_VIEW_CHECK, /* WCO on an auto-updatable view */ WCO_RLS_INSERT_CHECK, /* RLS INSERT WITH CHECK policy */ - WCO_RLS_UPDATE_CHECK /* RLS UPDATE WITH CHECK policy */ + WCO_RLS_UPDATE_CHECK, /* RLS UPDATE WITH CHECK policy */ + WCO_RLS_CONFLICT_CHECK /* RLS ON CONFLICT DO UPDATE USING policy */ } WCOKind; typedef struct WithCheckOption @@ -1026,6 +1029,37 @@ typedef struct WithClause } WithClause; /* + * InferClause - + * ON CONFLICT unique index inference clause + * + * Note: InferClause does not propagate into the Query representation. + */ +typedef struct InferClause +{ + NodeTag type; + List *indexElems; /* IndexElems to infer unique index */ + Node *whereClause; /* qualification (partial-index predicate) */ + char *conname; /* Constraint name, or NULL if unnamed */ + int location; /* token location, or -1 if unknown */ +} InferClause; + +/* + * OnConflictClause - + * representation of ON CONFLICT clause + * + * Note: OnConflictClause does not propagate into the Query representation. + */ +typedef struct OnConflictClause +{ + NodeTag type; + OnConflictAction action; /* DO NOTHING or UPDATE? */ + InferClause *infer; /* Optional index inference clause */ + List *targetList; /* the target list (of ResTarget) */ + Node *whereClause; /* qualifications */ + int location; /* token location, or -1 if unknown */ +} OnConflictClause; + +/* * CommonTableExpr - * representation of WITH list element * @@ -1075,6 +1109,7 @@ typedef struct InsertStmt RangeVar *relation; /* relation to insert into */ List *cols; /* optional: names of the target columns */ Node *selectStmt; /* the source SELECT/VALUES, or NULL */ + OnConflictClause *onConflictClause; /* ON CONFLICT clause */ List *returningList; /* list of expressions to return */ WithClause *withClause; /* WITH clause */ } InsertStmt; diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index baeba2d330f..c63492fa0be 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -45,6 +45,8 @@ typedef struct PlannedStmt bool hasModifyingCTE; /* has insert|update|delete in WITH? */ + bool isUpsert; /* is it insert ... ON CONFLICT UPDATE? */ + bool canSetTag; /* do I set the command result tag? */ bool transientPlan; /* redo plan when TransactionXmin changes? */ @@ -183,6 +185,12 @@ typedef struct ModifyTable List *fdwPrivLists; /* per-target-table FDW private data lists */ List *rowMarks; /* PlanRowMarks (non-locking only) */ int epqParam; /* ID of Param for EvalPlanQual re-eval */ + OnConflictAction onConflictAction; /* ON CONFLICT action */ + List *arbiterIndexes; /* List of ON CONFLICT arbiter index OIDs */ + List *onConflictSet; /* SET for INSERT ON CONFLICT DO UPDATE */ + Node *onConflictWhere;/* WHERE for ON CONFLICT UPDATE */ + Index exclRelRTI; /* RTI of the EXCLUDED pseudo relation */ + List *exclRelTlist; /* tlist of the EXCLUDED pseudo relation */ } ModifyTable; /* ---------------- diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index 4f1d234d307..8f2c64847e2 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -1143,6 +1143,22 @@ typedef struct CurrentOfExpr int cursor_param; /* refcursor parameter number, or 0 */ } CurrentOfExpr; +/* + * InferenceElem - an element of a unique index inference specification + * + * This mostly matches the structure of IndexElems, but having a dedicated + * primnode allows for a clean separation between the use of index parameters + * by utility commands, and this node. + */ +typedef struct InferenceElem +{ + Expr xpr; + Node *expr; /* expression to infer from, or NULL */ + Oid infercollid; /* OID of collation, or InvalidOid */ + Oid inferopfamily; /* OID of att opfamily, or InvalidOid */ + Oid inferopcinputtype; /* OID of att input type, or InvalidOid */ +} InferenceElem; + /*-------------------- * TargetEntry - * a target entry (used in query target lists) @@ -1307,4 +1323,30 @@ typedef struct FromExpr Node *quals; /* qualifiers on join, if any */ } FromExpr; +/*---------- + * OnConflictExpr - represents an ON CONFLICT DO ... expression + * + * The optimizer requires a list of inference elements, and optionally a WHERE + * clause to infer a unique index. The unique index (or, occasionally, + * indexes) inferred are used to arbitrate whether or not the alternative ON + * CONFLICT path is taken. + *---------- + */ +typedef struct OnConflictExpr +{ + NodeTag type; + OnConflictAction action; /* DO NOTHING or UPDATE? */ + + /* Arbiter */ + List *arbiterElems; /* unique index arbiter list (of InferenceElem's) */ + Node *arbiterWhere; /* unique index arbiter WHERE clause */ + Oid constraint; /* pg_constraint OID for arbiter */ + + /* ON CONFLICT UPDATE */ + List *onConflictSet; /* List of ON CONFLICT SET TargetEntrys */ + Node *onConflictWhere;/* qualifiers to restrict UPDATE to */ + int exclRelIndex; /* RT index of 'excluded' relation */ + List *exclRelTlist; /* tlist of the EXCLUDED pseudo relation */ +} OnConflictExpr; + #endif /* PRIMNODES_H */ diff --git a/src/include/optimizer/plancat.h b/src/include/optimizer/plancat.h index 8eb2e57d7b7..11e7d4d26bb 100644 --- a/src/include/optimizer/plancat.h +++ b/src/include/optimizer/plancat.h @@ -28,6 +28,8 @@ extern PGDLLIMPORT get_relation_info_hook_type get_relation_info_hook; extern void get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, RelOptInfo *rel); +extern List *infer_arbiter_indexes(PlannerInfo *root); + extern void estimate_rel_size(Relation rel, int32 *attr_widths, BlockNumber *pages, double *tuples, double *allvisfrac); diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index 0c8cbcded96..1d4ab0488e1 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -86,7 +86,7 @@ extern ModifyTable *make_modifytable(PlannerInfo *root, Index nominalRelation, List *resultRelations, List *subplans, List *withCheckOptionLists, List *returningLists, - List *rowMarks, int epqParam); + List *rowMarks, OnConflictExpr *onconflict, int epqParam); extern bool is_projection_capable_plan(Plan *plan); /* diff --git a/src/include/optimizer/prep.h b/src/include/optimizer/prep.h index 05e46c5b788..dcd078ee43d 100644 --- a/src/include/optimizer/prep.h +++ b/src/include/optimizer/prep.h @@ -45,6 +45,9 @@ extern void expand_security_quals(PlannerInfo *root, List *tlist); */ extern List *preprocess_targetlist(PlannerInfo *root, List *tlist); +extern List *preprocess_onconflict_targetlist(List *tlist, + int result_relation, List *range_table); + extern PlanRowMark *get_plan_rowmark(List *rowmarks, Index rtindex); /* diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 5b1ee154240..faea99108c4 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -87,6 +87,7 @@ PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD) PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD) PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD) PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD) +PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD) PG_KEYWORD("connection", CONNECTION, UNRESERVED_KEYWORD) PG_KEYWORD("constraint", CONSTRAINT, RESERVED_KEYWORD) PG_KEYWORD("constraints", CONSTRAINTS, UNRESERVED_KEYWORD) diff --git a/src/include/parser/parse_clause.h b/src/include/parser/parse_clause.h index 6a4438f5566..f1b7d3d8969 100644 --- a/src/include/parser/parse_clause.h +++ b/src/include/parser/parse_clause.h @@ -41,6 +41,10 @@ extern List *transformDistinctClause(ParseState *pstate, List **targetlist, List *sortClause, bool is_agg); extern List *transformDistinctOnClause(ParseState *pstate, List *distinctlist, List **targetlist, List *sortClause); +extern void transformOnConflictArbiter(ParseState *pstate, + OnConflictClause *onConflictClause, + List **arbiterExpr, Node **arbiterWhere, + Oid *constraint); extern List *addTargetToSortList(ParseState *pstate, TargetEntry *tle, List *sortlist, List *targetlist, SortBy *sortby, diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h index 6a5528a7344..928b1ca170f 100644 --- a/src/include/replication/reorderbuffer.h +++ b/src/include/replication/reorderbuffer.h @@ -43,6 +43,11 @@ typedef struct ReorderBufferTupleBuf * and ComboCids in the same list with the user visible INSERT/UPDATE/DELETE * changes. Users of the decoding facilities will never see changes with * *_INTERNAL_* actions. + * + * The INTERNAL_SPEC_INSERT and INTERNAL_SPEC_CONFIRM changes concern + * "speculative insertions", and their confirmation respectively. They're + * used by INSERT .. ON CONFLICT .. UPDATE. Users of logical decoding don't + * have to care about these. */ enum ReorderBufferChangeType { @@ -51,7 +56,9 @@ enum ReorderBufferChangeType REORDER_BUFFER_CHANGE_DELETE, REORDER_BUFFER_CHANGE_INTERNAL_SNAPSHOT, REORDER_BUFFER_CHANGE_INTERNAL_COMMAND_ID, - REORDER_BUFFER_CHANGE_INTERNAL_TUPLECID + REORDER_BUFFER_CHANGE_INTERNAL_TUPLECID, + REORDER_BUFFER_CHANGE_INTERNAL_SPEC_INSERT, + REORDER_BUFFER_CHANGE_INTERNAL_SPEC_CONFIRM }; /* diff --git a/src/include/rewrite/rowsecurity.h b/src/include/rewrite/rowsecurity.h index 115c9a8e430..eb4b20559f5 100644 --- a/src/include/rewrite/rowsecurity.h +++ b/src/include/rewrite/rowsecurity.h @@ -41,7 +41,8 @@ extern PGDLLIMPORT row_security_policy_hook_type row_security_policy_hook_permis extern PGDLLIMPORT row_security_policy_hook_type row_security_policy_hook_restrictive; -extern void get_row_security_policies(Query* root, RangeTblEntry* rte, int rt_index, +extern void get_row_security_policies(Query *root, CmdType commandType, + RangeTblEntry *rte, int rt_index, List **securityQuals, List **withCheckOptions, bool *hasRowSecurity, bool *hasSubLinks); diff --git a/src/include/storage/lmgr.h b/src/include/storage/lmgr.h index f5d70e5141e..7cc75fc1065 100644 --- a/src/include/storage/lmgr.h +++ b/src/include/storage/lmgr.h @@ -76,6 +76,11 @@ extern bool ConditionalXactLockTableWait(TransactionId xid); extern void WaitForLockers(LOCKTAG heaplocktag, LOCKMODE lockmode); extern void WaitForLockersMultiple(List *locktags, LOCKMODE lockmode); +/* Lock an XID for tuple insertion (used to wait for an insertion to finish) */ +extern uint32 SpeculativeInsertionLockAcquire(TransactionId xid); +extern void SpeculativeInsertionLockRelease(TransactionId xid); +extern void SpeculativeInsertionWait(TransactionId xid, uint32 token); + /* Lock a general object (other than a relation) of the current database */ extern void LockDatabaseObject(Oid classid, Oid objid, uint16 objsubid, LOCKMODE lockmode); diff --git a/src/include/storage/lock.h b/src/include/storage/lock.h index dae517f3fe0..b4eb1b4a9e3 100644 --- a/src/include/storage/lock.h +++ b/src/include/storage/lock.h @@ -176,6 +176,8 @@ typedef enum LockTagType /* ID info for a transaction is its TransactionId */ LOCKTAG_VIRTUALTRANSACTION, /* virtual transaction (ditto) */ /* ID info for a virtual transaction is its VirtualTransactionId */ + LOCKTAG_SPECULATIVE_TOKEN, /* speculative insertion Xid and token */ + /* ID info for a transaction is its TransactionId */ LOCKTAG_OBJECT, /* non-relation database object */ /* ID info for an object is DB OID + CLASS OID + OBJECT OID + SUBID */ @@ -261,6 +263,14 @@ typedef struct LOCKTAG (locktag).locktag_type = LOCKTAG_VIRTUALTRANSACTION, \ (locktag).locktag_lockmethodid = DEFAULT_LOCKMETHOD) +#define SET_LOCKTAG_SPECULATIVE_INSERTION(locktag,xid,token) \ + ((locktag).locktag_field1 = (xid), \ + (locktag).locktag_field2 = (token), \ + (locktag).locktag_field3 = 0, \ + (locktag).locktag_field4 = 0, \ + (locktag).locktag_type = LOCKTAG_SPECULATIVE_TOKEN, \ + (locktag).locktag_lockmethodid = DEFAULT_LOCKMETHOD) + #define SET_LOCKTAG_OBJECT(locktag,dboid,classoid,objoid,objsubid) \ ((locktag).locktag_field1 = (dboid), \ (locktag).locktag_field2 = (classoid), \ diff --git a/src/include/utils/snapshot.h b/src/include/utils/snapshot.h index 26fb2573c71..a734bf00752 100644 --- a/src/include/utils/snapshot.h +++ b/src/include/utils/snapshot.h @@ -69,31 +69,41 @@ typedef struct SnapshotData * progress, unless the snapshot was taken during recovery in which case * it's empty. For historic MVCC snapshots, the meaning is inverted, i.e. * it contains *committed* transactions between xmin and xmax. + * + * note: all ids in xip[] satisfy xmin <= xip[i] < xmax */ TransactionId *xip; uint32 xcnt; /* # of xact ids in xip[] */ - /* note: all ids in xip[] satisfy xmin <= xip[i] < xmax */ - int32 subxcnt; /* # of xact ids in subxip[] */ /* * For non-historic MVCC snapshots, this contains subxact IDs that are in * progress (and other transactions that are in progress if taken during * recovery). For historic snapshot it contains *all* xids assigned to the * replayed transaction, including the toplevel xid. + * + * note: all ids in subxip[] are >= xmin, but we don't bother filtering + * out any that are >= xmax */ TransactionId *subxip; + int32 subxcnt; /* # of xact ids in subxip[] */ bool suboverflowed; /* has the subxip array overflowed? */ + bool takenDuringRecovery; /* recovery-shaped snapshot? */ bool copied; /* false if it's a static snapshot */ + CommandId curcid; /* in my xact, CID < curcid are visible */ + /* - * note: all ids in subxip[] are >= xmin, but we don't bother filtering - * out any that are >= xmax + * An extra return value for HeapTupleSatisfiesDirty, not used in MVCC + * snapshots. + */ + uint32 speculativeToken; + + /* + * Book-keeping information, used by the snapshot manager */ - CommandId curcid; /* in my xact, CID < curcid are visible */ uint32 active_count; /* refcount on ActiveSnapshot stack */ uint32 regd_count; /* refcount on RegisteredSnapshots */ - pairingheap_node ph_node; /* link in the RegisteredSnapshots heap */ } SnapshotData; diff --git a/src/test/isolation/expected/insert-conflict-do-nothing.out b/src/test/isolation/expected/insert-conflict-do-nothing.out new file mode 100644 index 00000000000..0a0958f0341 --- /dev/null +++ b/src/test/isolation/expected/insert-conflict-do-nothing.out @@ -0,0 +1,23 @@ +Parsed test spec with 2 sessions + +starting permutation: donothing1 donothing2 c1 select2 c2 +step donothing1: INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; +step donothing2: INSERT INTO ints(key, val) VALUES(1, 'donothing2') ON CONFLICT DO NOTHING; <waiting ...> +step c1: COMMIT; +step donothing2: <... completed> +step select2: SELECT * FROM ints; +key val + +1 donothing1 +step c2: COMMIT; + +starting permutation: donothing1 donothing2 a1 select2 c2 +step donothing1: INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; +step donothing2: INSERT INTO ints(key, val) VALUES(1, 'donothing2') ON CONFLICT DO NOTHING; <waiting ...> +step a1: ABORT; +step donothing2: <... completed> +step select2: SELECT * FROM ints; +key val + +1 donothing2 +step c2: COMMIT; diff --git a/src/test/isolation/expected/insert-conflict-do-update-2.out b/src/test/isolation/expected/insert-conflict-do-update-2.out new file mode 100644 index 00000000000..05fb06f8d8d --- /dev/null +++ b/src/test/isolation/expected/insert-conflict-do-update-2.out @@ -0,0 +1,23 @@ +Parsed test spec with 2 sessions + +starting permutation: insert1 insert2 c1 select2 c2 +step insert1: INSERT INTO upsert(key, payload) VALUES('FooFoo', 'insert1') ON CONFLICT (lower(key)) DO UPDATE set key = EXCLUDED.key, payload = upsert.payload || ' updated by insert1'; +step insert2: INSERT INTO upsert(key, payload) VALUES('FOOFOO', 'insert2') ON CONFLICT (lower(key)) DO UPDATE set key = EXCLUDED.key, payload = upsert.payload || ' updated by insert2'; <waiting ...> +step c1: COMMIT; +step insert2: <... completed> +step select2: SELECT * FROM upsert; +key payload + +FOOFOO insert1 updated by insert2 +step c2: COMMIT; + +starting permutation: insert1 insert2 a1 select2 c2 +step insert1: INSERT INTO upsert(key, payload) VALUES('FooFoo', 'insert1') ON CONFLICT (lower(key)) DO UPDATE set key = EXCLUDED.key, payload = upsert.payload || ' updated by insert1'; +step insert2: INSERT INTO upsert(key, payload) VALUES('FOOFOO', 'insert2') ON CONFLICT (lower(key)) DO UPDATE set key = EXCLUDED.key, payload = upsert.payload || ' updated by insert2'; <waiting ...> +step a1: ABORT; +step insert2: <... completed> +step select2: SELECT * FROM upsert; +key payload + +FOOFOO insert2 +step c2: COMMIT; diff --git a/src/test/isolation/expected/insert-conflict-do-update-3.out b/src/test/isolation/expected/insert-conflict-do-update-3.out new file mode 100644 index 00000000000..66004106183 --- /dev/null +++ b/src/test/isolation/expected/insert-conflict-do-update-3.out @@ -0,0 +1,26 @@ +Parsed test spec with 2 sessions + +starting permutation: update2 insert1 c2 select1surprise c1 +step update2: UPDATE colors SET is_active = true WHERE key = 1; +step insert1: + WITH t AS ( + INSERT INTO colors(key, color, is_active) + VALUES(1, 'Brown', true), (2, 'Gray', true) + ON CONFLICT (key) DO UPDATE + SET color = EXCLUDED.color + WHERE colors.is_active) + SELECT * FROM colors ORDER BY key; <waiting ...> +step c2: COMMIT; +step insert1: <... completed> +key color is_active + +1 Red f +2 Green f +3 Blue f +step select1surprise: SELECT * FROM colors ORDER BY key; +key color is_active + +1 Brown t +2 Green f +3 Blue f +step c1: COMMIT; diff --git a/src/test/isolation/expected/insert-conflict-do-update.out b/src/test/isolation/expected/insert-conflict-do-update.out new file mode 100644 index 00000000000..a6349187842 --- /dev/null +++ b/src/test/isolation/expected/insert-conflict-do-update.out @@ -0,0 +1,23 @@ +Parsed test spec with 2 sessions + +starting permutation: insert1 insert2 c1 select2 c2 +step insert1: INSERT INTO upsert(key, val) VALUES(1, 'insert1') ON CONFLICT (key) DO UPDATE set val = upsert.val || ' updated by insert1'; +step insert2: INSERT INTO upsert(key, val) VALUES(1, 'insert2') ON CONFLICT (key) DO UPDATE set val = upsert.val || ' updated by insert2'; <waiting ...> +step c1: COMMIT; +step insert2: <... completed> +step select2: SELECT * FROM upsert; +key val + +1 insert1 updated by insert2 +step c2: COMMIT; + +starting permutation: insert1 insert2 a1 select2 c2 +step insert1: INSERT INTO upsert(key, val) VALUES(1, 'insert1') ON CONFLICT (key) DO UPDATE set val = upsert.val || ' updated by insert1'; +step insert2: INSERT INTO upsert(key, val) VALUES(1, 'insert2') ON CONFLICT (key) DO UPDATE set val = upsert.val || ' updated by insert2'; <waiting ...> +step a1: ABORT; +step insert2: <... completed> +step select2: SELECT * FROM upsert; +key val + +1 insert2 +step c2: COMMIT; diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 3e2614ecacd..c0ed637cd24 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -16,6 +16,10 @@ test: fk-deadlock2 test: eval-plan-qual test: lock-update-delete test: lock-update-traversal +test: insert-conflict-do-nothing +test: insert-conflict-do-update +test: insert-conflict-do-update-2 +test: insert-conflict-do-update-3 test: delete-abort-savept test: delete-abort-savept-2 test: aborted-keyrevoke diff --git a/src/test/isolation/specs/insert-conflict-do-nothing.spec b/src/test/isolation/specs/insert-conflict-do-nothing.spec new file mode 100644 index 00000000000..9b92c35cec6 --- /dev/null +++ b/src/test/isolation/specs/insert-conflict-do-nothing.spec @@ -0,0 +1,41 @@ +# INSERT...ON CONFLICT DO NOTHING test +# +# This test tries to expose problems with the interaction between concurrent +# sessions during INSERT...ON CONFLICT DO NOTHING. +# +# The convention here is that session 1 always ends up inserting, and session 2 +# always ends up doing nothing. + +setup +{ + CREATE TABLE ints (key int primary key, val text); +} + +teardown +{ + DROP TABLE ints; +} + +session "s1" +setup +{ + BEGIN ISOLATION LEVEL READ COMMITTED; +} +step "donothing1" { INSERT INTO ints(key, val) VALUES(1, 'donothing1') ON CONFLICT DO NOTHING; } +step "c1" { COMMIT; } +step "a1" { ABORT; } + +session "s2" +setup +{ + BEGIN ISOLATION LEVEL READ COMMITTED; +} +step "donothing2" { INSERT INTO ints(key, val) VALUES(1, 'donothing2') ON CONFLICT DO NOTHING; } +step "select2" { SELECT * FROM ints; } +step "c2" { COMMIT; } +step "a2" { ABORT; } + +# Regular case where one session block-waits on another to determine if it +# should proceed with an insert or do nothing. +permutation "donothing1" "donothing2" "c1" "select2" "c2" +permutation "donothing1" "donothing2" "a1" "select2" "c2" diff --git a/src/test/isolation/specs/insert-conflict-do-update-2.spec b/src/test/isolation/specs/insert-conflict-do-update-2.spec new file mode 100644 index 00000000000..cd7e3f42feb --- /dev/null +++ b/src/test/isolation/specs/insert-conflict-do-update-2.spec @@ -0,0 +1,41 @@ +# INSERT...ON CONFLICT DO UPDATE test +# +# This test shows a plausible scenario in which the user might wish to UPDATE a +# value that is also constrained by the unique index that is the arbiter of +# whether the alternative path should be taken. + +setup +{ + CREATE TABLE upsert (key text not null, payload text); + CREATE UNIQUE INDEX ON upsert(lower(key)); +} + +teardown +{ + DROP TABLE upsert; +} + +session "s1" +setup +{ + BEGIN ISOLATION LEVEL READ COMMITTED; +} +step "insert1" { INSERT INTO upsert(key, payload) VALUES('FooFoo', 'insert1') ON CONFLICT (lower(key)) DO UPDATE set key = EXCLUDED.key, payload = upsert.payload || ' updated by insert1'; } +step "c1" { COMMIT; } +step "a1" { ABORT; } + +session "s2" +setup +{ + BEGIN ISOLATION LEVEL READ COMMITTED; +} +step "insert2" { INSERT INTO upsert(key, payload) VALUES('FOOFOO', 'insert2') ON CONFLICT (lower(key)) DO UPDATE set key = EXCLUDED.key, payload = upsert.payload || ' updated by insert2'; } +step "select2" { SELECT * FROM upsert; } +step "c2" { COMMIT; } +step "a2" { ABORT; } + +# One session (session 2) block-waits on another (session 1) to determine if it +# should proceed with an insert or update. The user can still usefully UPDATE +# a column constrained by a unique index, as the example illustrates. +permutation "insert1" "insert2" "c1" "select2" "c2" +permutation "insert1" "insert2" "a1" "select2" "c2" diff --git a/src/test/isolation/specs/insert-conflict-do-update-3.spec b/src/test/isolation/specs/insert-conflict-do-update-3.spec new file mode 100644 index 00000000000..e282c3beca5 --- /dev/null +++ b/src/test/isolation/specs/insert-conflict-do-update-3.spec @@ -0,0 +1,69 @@ +# INSERT...ON CONFLICT DO UPDATE test +# +# Other INSERT...ON CONFLICT DO UPDATE isolation tests illustrate the "MVCC +# violation" added to facilitate the feature, whereby a +# not-visible-to-our-snapshot tuple can be updated by our command all the same. +# This is generally needed to provide a guarantee of a successful INSERT or +# UPDATE in READ COMMITTED mode. This MVCC violation is quite distinct from +# the putative "MVCC violation" that has existed in PostgreSQL for many years, +# the EvalPlanQual() mechanism, because that mechanism always starts from a +# tuple that is visible to the command's MVCC snapshot. This test illustrates +# a slightly distinct user-visible consequence of the same MVCC violation +# generally associated with INSERT...ON CONFLICT DO UPDATE. The impact of the +# MVCC violation goes a little beyond updating MVCC-invisible tuples. +# +# With INSERT...ON CONFLICT DO UPDATE, the UPDATE predicate is only evaluated +# once, on this conclusively-locked tuple, and not any other version of the +# same tuple. It is therefore possible (in READ COMMITTED mode) that the +# predicate "fail to be satisfied" according to the command's MVCC snapshot. +# It might simply be that there is no row version visible, but it's also +# possible that there is some row version visible, but only as a version that +# doesn't satisfy the predicate. If, however, the conclusively-locked version +# satisfies the predicate, that's good enough, and the tuple is updated. The +# MVCC-snapshot-visible row version is denied the opportunity to prevent the +# UPDATE from taking place, because we don't walk the UPDATE chain in the usual +# way. + +setup +{ + CREATE TABLE colors (key int4 PRIMARY KEY, color text, is_active boolean); + INSERT INTO colors (key, color, is_active) VALUES(1, 'Red', false); + INSERT INTO colors (key, color, is_active) VALUES(2, 'Green', false); + INSERT INTO colors (key, color, is_active) VALUES(3, 'Blue', false); +} + +teardown +{ + DROP TABLE colors; +} + +session "s1" +setup +{ + BEGIN ISOLATION LEVEL READ COMMITTED; +} +step "insert1" { + WITH t AS ( + INSERT INTO colors(key, color, is_active) + VALUES(1, 'Brown', true), (2, 'Gray', true) + ON CONFLICT (key) DO UPDATE + SET color = EXCLUDED.color + WHERE colors.is_active) + SELECT * FROM colors ORDER BY key;} +step "select1surprise" { SELECT * FROM colors ORDER BY key; } +step "c1" { COMMIT; } + +session "s2" +setup +{ + BEGIN ISOLATION LEVEL READ COMMITTED; +} +step "update2" { UPDATE colors SET is_active = true WHERE key = 1; } +step "c2" { COMMIT; } + +# Perhaps surprisingly, the session 1 MVCC-snapshot-visible tuple (the tuple +# with the pre-populated color 'Red') is denied the opportunity to prevent the +# UPDATE from taking place -- only the conclusively-locked tuple version +# matters, and so the tuple with key value 1 was updated to 'Brown' (but not +# tuple with key value 2, since nothing changed there): +permutation "update2" "insert1" "c2" "select1surprise" "c1" diff --git a/src/test/isolation/specs/insert-conflict-do-update.spec b/src/test/isolation/specs/insert-conflict-do-update.spec new file mode 100644 index 00000000000..5d335a34449 --- /dev/null +++ b/src/test/isolation/specs/insert-conflict-do-update.spec @@ -0,0 +1,40 @@ +# INSERT...ON CONFLICT DO UPDATE test +# +# This test tries to expose problems with the interaction between concurrent +# sessions. + +setup +{ + CREATE TABLE upsert (key int primary key, val text); +} + +teardown +{ + DROP TABLE upsert; +} + +session "s1" +setup +{ + BEGIN ISOLATION LEVEL READ COMMITTED; +} +step "insert1" { INSERT INTO upsert(key, val) VALUES(1, 'insert1') ON CONFLICT (key) DO UPDATE set val = upsert.val || ' updated by insert1'; } +step "c1" { COMMIT; } +step "a1" { ABORT; } + +session "s2" +setup +{ + BEGIN ISOLATION LEVEL READ COMMITTED; +} +step "insert2" { INSERT INTO upsert(key, val) VALUES(1, 'insert2') ON CONFLICT (key) DO UPDATE set val = upsert.val || ' updated by insert2'; } +step "select2" { SELECT * FROM upsert; } +step "c2" { COMMIT; } +step "a2" { ABORT; } + +# One session (session 2) block-waits on another (session 1) to determine if it +# should proceed with an insert or update. Notably, this entails updating a +# tuple while there is no version of that tuple visible to the updating +# session's snapshot. This is permitted only in READ COMMITTED mode. +permutation "insert1" "insert2" "c1" "select2" "c2" +permutation "insert1" "insert2" "a1" "select2" "c2" diff --git a/src/test/regress/expected/errors.out b/src/test/regress/expected/errors.out index 5f8868da26e..210e5ff39cb 100644 --- a/src/test/regress/expected/errors.out +++ b/src/test/regress/expected/errors.out @@ -32,7 +32,9 @@ LINE 1: select nonesuch from pg_database; ^ -- empty distinct list isn't OK select distinct from pg_database; -ERROR: SELECT DISTINCT must have at least one column +ERROR: syntax error at or near "from" +LINE 1: select distinct from pg_database; + ^ -- bad attribute name on lhs of operator select * from pg_database where nonesuch = pg_database.datname; ERROR: column "nonesuch" does not exist diff --git a/src/test/regress/expected/insert_conflict.out b/src/test/regress/expected/insert_conflict.out new file mode 100644 index 00000000000..3273d987932 --- /dev/null +++ b/src/test/regress/expected/insert_conflict.out @@ -0,0 +1,476 @@ +-- +-- insert...on conflict do unique index inference +-- +create table insertconflicttest(key int4, fruit text); +-- +-- Test unique index inference with operator class specifications and +-- named collations +-- +create unique index op_index_key on insertconflicttest(key, fruit text_pattern_ops); +create unique index collation_index_key on insertconflicttest(key, fruit collate "C"); +create unique index both_index_key on insertconflicttest(key, fruit collate "C" text_pattern_ops); +create unique index both_index_expr_key on insertconflicttest(key, lower(fruit) collate "C" text_pattern_ops); +-- fails +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (key) do nothing; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (fruit) do nothing; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +-- succeeds +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (key, fruit) do nothing; + QUERY PLAN +------------------------------------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: NOTHING + Conflict Arbiter Indexes: op_index_key, collation_index_key, both_index_key + -> Result +(4 rows) + +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (fruit, key, fruit, key) do nothing; + QUERY PLAN +------------------------------------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: NOTHING + Conflict Arbiter Indexes: op_index_key, collation_index_key, both_index_key + -> Result +(4 rows) + +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (lower(fruit), key, lower(fruit), key) do nothing; + QUERY PLAN +------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: NOTHING + Conflict Arbiter Indexes: both_index_expr_key + -> Result +(4 rows) + +-- Neither collation nor operator class specifications are required -- +-- supplying them merely *limits* matches to indexes with matching opclasses +-- used for relevant indexes +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (key, fruit text_pattern_ops) do nothing; + QUERY PLAN +---------------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: NOTHING + Conflict Arbiter Indexes: op_index_key, both_index_key + -> Result +(4 rows) + +-- Okay, arbitrates using both index where text_pattern_ops opclass does and +-- does not appear. +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (key, fruit collate "C") do nothing; + QUERY PLAN +----------------------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: NOTHING + Conflict Arbiter Indexes: collation_index_key, both_index_key + -> Result +(4 rows) + +-- Okay, but only accepts the single index where both opclass and collation are +-- specified +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (fruit collate "C" text_pattern_ops, key) do nothing; + QUERY PLAN +-------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: NOTHING + Conflict Arbiter Indexes: both_index_key + -> Result +(4 rows) + +-- Okay, but only accepts the single index where both opclass and collation are +-- specified (plus expression variant) +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (lower(fruit) collate "C", key, key) do nothing; + QUERY PLAN +------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: NOTHING + Conflict Arbiter Indexes: both_index_expr_key + -> Result +(4 rows) + +-- Attribute appears twice, while not all attributes/expressions on attributes +-- appearing within index definition match in terms of both opclass and +-- collation. +-- +-- Works because every attribute in inference specification needs to be +-- satisfied once or more by cataloged index attribute, and as always when an +-- attribute in the cataloged definition has a non-default opclass/collation, +-- it still satisfied some inference attribute lacking any particular +-- opclass/collation specification. +-- +-- The implementation is liberal in accepting inference specifications on the +-- assumption that multiple inferred unique indexes will prevent problematic +-- cases. It rolls with unique indexes where attributes redundantly appear +-- multiple times, too (which is not tested here). +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (fruit, key, fruit text_pattern_ops, key) do nothing; + QUERY PLAN +---------------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: NOTHING + Conflict Arbiter Indexes: op_index_key, both_index_key + -> Result +(4 rows) + +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (lower(fruit) collate "C" text_pattern_ops, key, key) do nothing; + QUERY PLAN +------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: NOTHING + Conflict Arbiter Indexes: both_index_expr_key + -> Result +(4 rows) + +drop index op_index_key; +drop index collation_index_key; +drop index both_index_key; +drop index both_index_expr_key; +-- +-- Single key tests +-- +create unique index key_index on insertconflicttest(key); +-- +-- Explain tests +-- +explain (costs off) insert into insertconflicttest values (0, 'Bilberry') on conflict (key) do update set fruit = excluded.fruit; + QUERY PLAN +--------------------------------------- + Insert on insertconflicttest + Conflict Resolution: UPDATE + Conflict Arbiter Indexes: key_index + -> Result +(4 rows) + +-- Should display qual actually attributable to internal sequential scan: +explain (costs off) insert into insertconflicttest values (0, 'Bilberry') on conflict (key) do update set fruit = excluded.fruit where insertconflicttest.fruit != 'Cawesh'; + QUERY PLAN +----------------------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: UPDATE + Conflict Arbiter Indexes: key_index + Conflict Filter: (insertconflicttest.fruit <> 'Cawesh'::text) + -> Result +(5 rows) + +-- With EXCLUDED.* expression in scan node: +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (key) do update set fruit = excluded.fruit where excluded.fruit != 'Elderberry'; + QUERY PLAN +----------------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: UPDATE + Conflict Arbiter Indexes: key_index + Conflict Filter: (excluded.fruit <> 'Elderberry'::text) + -> Result +(5 rows) + +-- Does the same, but JSON format shows "Conflict Arbiter Index" as JSON array: +explain (costs off, format json) insert into insertconflicttest values (0, 'Bilberry') on conflict (key) do update set fruit = excluded.fruit where insertconflicttest.fruit != 'Lime' returning *; + QUERY PLAN +------------------------------------------------------------------------ + [ + + { + + "Plan": { + + "Node Type": "ModifyTable", + + "Operation": "Insert", + + "Relation Name": "insertconflicttest", + + "Alias": "insertconflicttest", + + "Conflict Resolution": "UPDATE", + + "Conflict Arbiter Indexes": ["key_index"], + + "Conflict Filter": "(insertconflicttest.fruit <> 'Lime'::text)",+ + "Plans": [ + + { + + "Node Type": "Result", + + "Parent Relationship": "Member" + + } + + ] + + } + + } + + ] +(1 row) + +-- Fails (no unique index inference specification, required for do update variant): +insert into insertconflicttest values (1, 'Apple') on conflict do update set fruit = excluded.fruit; +ERROR: ON CONFLICT DO UPDATE requires inference specification or constraint name +LINE 1: ...nsert into insertconflicttest values (1, 'Apple') on conflic... + ^ +HINT: For example, ON CONFLICT ON CONFLICT (<column>). +-- inference succeeds: +insert into insertconflicttest values (1, 'Apple') on conflict (key) do update set fruit = excluded.fruit; +insert into insertconflicttest values (2, 'Orange') on conflict (key, key, key) do update set fruit = excluded.fruit; +-- Succeed, since multi-assignment does not involve subquery: +insert into insertconflicttest +values (1, 'Apple'), (2, 'Orange') +on conflict (key) do update set (fruit, key) = (excluded.fruit, excluded.key); +-- Give good diagnostic message when EXCLUDED.* spuriously referenced from +-- RETURNING: +insert into insertconflicttest values (1, 'Apple') on conflict (key) do update set fruit = excluded.fruit RETURNING excluded.fruit; +ERROR: invalid reference to FROM-clause entry for table "excluded" +LINE 1: ...y) do update set fruit = excluded.fruit RETURNING excluded.f... + ^ +HINT: There is an entry for table "excluded", but it cannot be referenced from this part of the query. +-- Only suggest <table>.* column when inference element misspelled: +insert into insertconflicttest values (1, 'Apple') on conflict (keyy) do update set fruit = excluded.fruit; +ERROR: column "keyy" does not exist +LINE 1: ...nsertconflicttest values (1, 'Apple') on conflict (keyy) do ... + ^ +HINT: Perhaps you meant to reference the column "insertconflicttest"."key". +-- Have useful HINT for EXCLUDED.* RTE within UPDATE: +insert into insertconflicttest values (1, 'Apple') on conflict (key) do update set fruit = excluded.fruitt; +ERROR: column excluded.fruitt does not exist +LINE 1: ... 'Apple') on conflict (key) do update set fruit = excluded.f... + ^ +HINT: Perhaps you meant to reference the column "excluded"."fruit". +-- inference fails: +insert into insertconflicttest values (3, 'Kiwi') on conflict (key, fruit) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (4, 'Mango') on conflict (fruit, key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (5, 'Lemon') on conflict (fruit) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (6, 'Passionfruit') on conflict (lower(fruit)) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +-- Check the target relation can be aliased +insert into insertconflicttest values (6, 'Passionfruits') on conflict (key) do update set fruit = excluded.fruit; +insert into insertconflicttest AS ict values (6, 'Passionfruit') on conflict (key) do update set fruit = excluded.fruit; -- ok, no reference to target table +insert into insertconflicttest AS ict values (6, 'Passionfruit') on conflict (key) do update set fruit = ict.fruit; -- ok, alias +insert into insertconflicttest AS ict values (6, 'Passionfruit') on conflict (key) do update set fruit = insertconflicttest.fruit; -- error, references aliased away name +ERROR: invalid reference to FROM-clause entry for table "insertconflicttest" +LINE 1: ...onfruit') on conflict (key) do update set fruit = insertconf... + ^ +HINT: Perhaps you meant to reference the table alias "ict". +drop index key_index; +-- +-- Composite key tests +-- +create unique index comp_key_index on insertconflicttest(key, fruit); +-- inference succeeds: +insert into insertconflicttest values (7, 'Raspberry') on conflict (key, fruit) do update set fruit = excluded.fruit; +insert into insertconflicttest values (8, 'Lime') on conflict (fruit, key) do update set fruit = excluded.fruit; +-- inference fails: +insert into insertconflicttest values (9, 'Banana') on conflict (key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (10, 'Blueberry') on conflict (key, key, key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (11, 'Cherry') on conflict (key, lower(fruit)) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (12, 'Date') on conflict (lower(fruit), key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +drop index comp_key_index; +-- +-- Partial index tests, no inference predicate specificied +-- +create unique index part_comp_key_index on insertconflicttest(key, fruit) where key < 5; +create unique index expr_part_comp_key_index on insertconflicttest(key, lower(fruit)) where key < 5; +-- inference fails: +insert into insertconflicttest values (13, 'Grape') on conflict (key, fruit) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (14, 'Raisin') on conflict (fruit, key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (15, 'Cranberry') on conflict (key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (16, 'Melon') on conflict (key, key, key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (17, 'Mulberry') on conflict (key, lower(fruit)) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (18, 'Pineapple') on conflict (lower(fruit), key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +drop index part_comp_key_index; +drop index expr_part_comp_key_index; +-- +-- Expression index tests +-- +create unique index expr_key_index on insertconflicttest(lower(fruit)); +-- inference succeeds: +insert into insertconflicttest values (20, 'Quince') on conflict (lower(fruit)) do update set fruit = excluded.fruit; +insert into insertconflicttest values (21, 'Pomegranate') on conflict (lower(fruit), lower(fruit)) do update set fruit = excluded.fruit; +-- inference fails: +insert into insertconflicttest values (22, 'Apricot') on conflict (upper(fruit)) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (23, 'Blackberry') on conflict (fruit) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +drop index expr_key_index; +-- +-- Expression index tests (with regular column) +-- +create unique index expr_comp_key_index on insertconflicttest(key, lower(fruit)); +create unique index tricky_expr_comp_key_index on insertconflicttest(key, lower(fruit), upper(fruit)); +-- inference succeeds: +insert into insertconflicttest values (24, 'Plum') on conflict (key, lower(fruit)) do update set fruit = excluded.fruit; +insert into insertconflicttest values (25, 'Peach') on conflict (lower(fruit), key) do update set fruit = excluded.fruit; +-- Should not infer "tricky_expr_comp_key_index" index: +explain (costs off) insert into insertconflicttest values (26, 'Fig') on conflict (lower(fruit), key, lower(fruit), key) do update set fruit = excluded.fruit; + QUERY PLAN +------------------------------------------------- + Insert on insertconflicttest + Conflict Resolution: UPDATE + Conflict Arbiter Indexes: expr_comp_key_index + -> Result +(4 rows) + +-- inference fails: +insert into insertconflicttest values (27, 'Prune') on conflict (key, upper(fruit)) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (28, 'Redcurrant') on conflict (fruit, key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (29, 'Nectarine') on conflict (key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +drop index expr_comp_key_index; +drop index tricky_expr_comp_key_index; +-- +-- Non-spurious duplicate violation tests +-- +create unique index key_index on insertconflicttest(key); +create unique index fruit_index on insertconflicttest(fruit); +-- succeeds, since UPDATE happens to update "fruit" to existing value: +insert into insertconflicttest values (26, 'Fig') on conflict (key) do update set fruit = excluded.fruit; +-- fails, since UPDATE is to row with key value 26, and we're updating "fruit" +-- to a value that happens to exist in another row ('peach'): +insert into insertconflicttest values (26, 'Peach') on conflict (key) do update set fruit = excluded.fruit; +ERROR: duplicate key value violates unique constraint "fruit_index" +DETAIL: Key (fruit)=(Peach) already exists. +-- succeeds, since "key" isn't repeated/referenced in UPDATE, and "fruit" +-- arbitrates that statement updates existing "Fig" row: +insert into insertconflicttest values (25, 'Fig') on conflict (fruit) do update set fruit = excluded.fruit; +drop index key_index; +drop index fruit_index; +-- +-- Test partial unique index inference +-- +create unique index partial_key_index on insertconflicttest(key) where fruit like '%berry'; +-- Succeeds +insert into insertconflicttest values (23, 'Blackberry') on conflict (key) where fruit like '%berry' do update set fruit = excluded.fruit; +insert into insertconflicttest values (23, 'Blackberry') on conflict (key) where fruit like '%berry' and fruit = 'inconsequential' do nothing; +-- fails +insert into insertconflicttest values (23, 'Blackberry') on conflict (key) do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (23, 'Blackberry') on conflict (key) where fruit like '%berry' or fruit = 'consequential' do nothing; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +insert into insertconflicttest values (23, 'Blackberry') on conflict (fruit) where fruit like '%berry' do update set fruit = excluded.fruit; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +drop index partial_key_index; +-- Cleanup +drop table insertconflicttest; +-- ****************************************************************** +-- * * +-- * Test inheritance (example taken from tutorial) * +-- * * +-- ****************************************************************** +create table cities ( + name text, + population float8, + altitude int -- (in ft) +); +create table capitals ( + state char(2) +) inherits (cities); +-- Create unique indexes. Due to a general limitation of inheritance, +-- uniqueness is only enforced per-relation. Unique index inference +-- specification will do the right thing, though. +create unique index cities_names_unique on cities (name); +create unique index capitals_names_unique on capitals (name); +-- prepopulate the tables. +insert into cities values ('San Francisco', 7.24E+5, 63); +insert into cities values ('Las Vegas', 2.583E+5, 2174); +insert into cities values ('Mariposa', 1200, 1953); +insert into capitals values ('Sacramento', 3.694E+5, 30, 'CA'); +insert into capitals values ('Madison', 1.913E+5, 845, 'WI'); +-- Tests proper for inheritance: +select * from capitals; + name | population | altitude | state +------------+------------+----------+------- + Sacramento | 369400 | 30 | CA + Madison | 191300 | 845 | WI +(2 rows) + +-- Succeeds: +insert into cities values ('Las Vegas', 2.583E+5, 2174) on conflict do nothing; +insert into capitals values ('Sacramento', 4664.E+5, 30, 'CA') on conflict (name) do update set population = excluded.population; +-- Wrong "Sacramento", so do nothing: +insert into capitals values ('Sacramento', 50, 2267, 'NE') on conflict (name) do nothing; +select * from capitals; + name | population | altitude | state +------------+------------+----------+------- + Madison | 191300 | 845 | WI + Sacramento | 466400000 | 30 | CA +(2 rows) + +insert into cities values ('Las Vegas', 5.83E+5, 2001) on conflict (name) do update set population = excluded.population, altitude = excluded.altitude; +select tableoid::regclass, * from cities; + tableoid | name | population | altitude +----------+---------------+------------+---------- + cities | San Francisco | 724000 | 63 + cities | Mariposa | 1200 | 1953 + cities | Las Vegas | 583000 | 2001 + capitals | Madison | 191300 | 845 + capitals | Sacramento | 466400000 | 30 +(5 rows) + +insert into capitals values ('Las Vegas', 5.83E+5, 2222, 'NV') on conflict (name) do update set population = excluded.population; +-- Capitals will contain new capital, Las Vegas: +select * from capitals; + name | population | altitude | state +------------+------------+----------+------- + Madison | 191300 | 845 | WI + Sacramento | 466400000 | 30 | CA + Las Vegas | 583000 | 2222 | NV +(3 rows) + +-- Cities contains two instances of "Las Vegas", since unique constraints don't +-- work across inheritance: +select tableoid::regclass, * from cities; + tableoid | name | population | altitude +----------+---------------+------------+---------- + cities | San Francisco | 724000 | 63 + cities | Mariposa | 1200 | 1953 + cities | Las Vegas | 583000 | 2001 + capitals | Madison | 191300 | 845 + capitals | Sacramento | 466400000 | 30 + capitals | Las Vegas | 583000 | 2222 +(6 rows) + +-- This only affects "cities" version of "Las Vegas": +insert into cities values ('Las Vegas', 5.86E+5, 2223) on conflict (name) do update set population = excluded.population, altitude = excluded.altitude; +select tableoid::regclass, * from cities; + tableoid | name | population | altitude +----------+---------------+------------+---------- + cities | San Francisco | 724000 | 63 + cities | Mariposa | 1200 | 1953 + cities | Las Vegas | 586000 | 2223 + capitals | Madison | 191300 | 845 + capitals | Sacramento | 466400000 | 30 + capitals | Las Vegas | 583000 | 2222 +(6 rows) + +-- clean up +drop table capitals; +drop table cities; +-- Make sure a table named excluded is handled properly +create table excluded(key int primary key, data text); +insert into excluded values(1, '1'); +-- error, ambiguous +insert into excluded values(1, '2') on conflict (key) do update set data = excluded.data RETURNING *; +ERROR: table reference "excluded" is ambiguous +LINE 1: ...es(1, '2') on conflict (key) do update set data = excluded.d... + ^ +-- ok, aliased +insert into excluded AS target values(1, '2') on conflict (key) do update set data = excluded.data RETURNING *; + key | data +-----+------ + 1 | 2 +(1 row) + +-- ok, aliased +insert into excluded AS target values(1, '2') on conflict (key) do update set data = target.data RETURNING *; + key | data +-----+------ + 1 | 2 +(1 row) + +-- make sure excluded isn't a problem in returning clause +insert into excluded values(1, '2') on conflict (key) do update set data = 3 RETURNING excluded.*; + key | data +-----+------ + 1 | 3 +(1 row) + +-- clean up +drop table excluded; diff --git a/src/test/regress/expected/privileges.out b/src/test/regress/expected/privileges.out index 0db1df30402..64a93309ebc 100644 --- a/src/test/regress/expected/privileges.out +++ b/src/test/regress/expected/privileges.out @@ -269,7 +269,7 @@ SELECT * FROM atestv2; -- fail (even though regressuser2 can access underlying a ERROR: permission denied for relation atest2 -- Test column level permissions SET SESSION AUTHORIZATION regressuser1; -CREATE TABLE atest5 (one int, two int, three int); +CREATE TABLE atest5 (one int, two int unique, three int, four int unique); CREATE TABLE atest6 (one int, two int, blue int); GRANT SELECT (one), INSERT (two), UPDATE (three) ON atest5 TO regressuser4; GRANT ALL (one) ON atest5 TO regressuser3; @@ -367,6 +367,33 @@ UPDATE atest5 SET one = 8; -- fail ERROR: permission denied for relation atest5 UPDATE atest5 SET three = 5, one = 2; -- fail ERROR: permission denied for relation atest5 +-- Check that column level privs are enforced in RETURNING +-- Ok. +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = 10; +-- Error. No SELECT on column three. +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = 10 RETURNING atest5.three; +ERROR: permission denied for relation atest5 +-- Ok. May SELECT on column "one": +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = 10 RETURNING atest5.one; + one +----- + +(1 row) + +-- Check that column level privileges are enforced for EXCLUDED +-- Ok. we may select one +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = EXCLUDED.one; +-- Error. No select rights on three +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = EXCLUDED.three; +ERROR: permission denied for relation atest5 +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set one = 8; -- fails (due to UPDATE) +ERROR: permission denied for relation atest5 +INSERT INTO atest5(three) VALUES (4) ON CONFLICT (two) DO UPDATE set three = 10; -- fails (due to INSERT) +ERROR: permission denied for relation atest5 +-- Check that the the columns in the inference require select privileges +-- Error. No privs on four +INSERT INTO atest5(three) VALUES (4) ON CONFLICT (four) DO UPDATE set three = 10; +ERROR: permission denied for relation atest5 SET SESSION AUTHORIZATION regressuser1; REVOKE ALL (one) ON atest5 FROM regressuser4; GRANT SELECT (one,two,blue) ON atest6 TO regressuser4; diff --git a/src/test/regress/expected/returning.out b/src/test/regress/expected/returning.out index 69bdacc103c..cb51bb86876 100644 --- a/src/test/regress/expected/returning.out +++ b/src/test/regress/expected/returning.out @@ -331,3 +331,27 @@ SELECT * FROM voo; 17 | zoo2 (2 rows) +-- Check aliased target relation +INSERT INTO foo AS bar DEFAULT VALUES RETURNING *; -- ok + f1 | f2 | f3 | f4 +----+----+----+---- + 4 | | 42 | 99 +(1 row) + +INSERT INTO foo AS bar DEFAULT VALUES RETURNING foo.*; -- fails, wrong name +ERROR: invalid reference to FROM-clause entry for table "foo" +LINE 1: INSERT INTO foo AS bar DEFAULT VALUES RETURNING foo.*; + ^ +HINT: Perhaps you meant to reference the table alias "bar". +INSERT INTO foo AS bar DEFAULT VALUES RETURNING bar.*; -- ok + f1 | f2 | f3 | f4 +----+----+----+---- + 5 | | 42 | 99 +(1 row) + +INSERT INTO foo AS bar DEFAULT VALUES RETURNING bar.f3; -- ok + f3 +---- + 42 +(1 row) + diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out index ad936321749..82bc47895a8 100644 --- a/src/test/regress/expected/rowsecurity.out +++ b/src/test/regress/expected/rowsecurity.out @@ -1491,6 +1491,138 @@ SELECT * FROM b1; (21 rows) -- +-- INSERT ... ON CONFLICT DO UPDATE and Row-level security +-- +SET SESSION AUTHORIZATION rls_regress_user0; +DROP POLICY p1 ON document; +CREATE POLICY p1 ON document FOR SELECT USING (true); +CREATE POLICY p2 ON document FOR INSERT WITH CHECK (dauthor = current_user); +CREATE POLICY p3 ON document FOR UPDATE + USING (cid = (SELECT cid from category WHERE cname = 'novel')) + WITH CHECK (dauthor = current_user); +SET SESSION AUTHORIZATION rls_regress_user1; +-- Exists... +SELECT * FROM document WHERE did = 2; + did | cid | dlevel | dauthor | dtitle +-----+-----+--------+-------------------+----------------- + 2 | 11 | 2 | rls_regress_user1 | my second novel +(1 row) + +-- ...so violates actual WITH CHECK OPTION within UPDATE (not INSERT, since +-- alternative UPDATE path happens to be taken): +INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user2', 'my first novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, dauthor = EXCLUDED.dauthor; +ERROR: new row violates row level security policy for "document" +-- Violates USING qual for UPDATE policy p3. +-- +-- UPDATE path is taken, but UPDATE fails purely because *existing* row to be +-- updated is not a "novel"/cid 11 (row is not leaked, even though we have +-- SELECT privileges sufficient to see the row in this instance): +INSERT INTO document VALUES (33, 22, 1, 'rls_regress_user1', 'okay science fiction'); -- preparation for next statement +INSERT INTO document VALUES (33, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'Some novel, replaces sci-fi') -- takes UPDATE path + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; +ERROR: new row violates row level security policy (USING expression) for "document" +-- Fine (we UPDATE, since INSERT WCOs and UPDATE security barrier quals + WCOs +-- not violated): +INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'my first novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; + did | cid | dlevel | dauthor | dtitle +-----+-----+--------+-------------------+---------------- + 2 | 11 | 2 | rls_regress_user1 | my first novel +(1 row) + +-- Fine (we INSERT, so "cid = 33" ("technology") isn't evaluated): +INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'some technology novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; + did | cid | dlevel | dauthor | dtitle +-----+-----+--------+-------------------+----------------------- + 78 | 11 | 1 | rls_regress_user1 | some technology novel +(1 row) + +-- Fine (same query, but we UPDATE, so "cid = 33", ("technology") is not the +-- case in respect of *existing* tuple): +INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'some technology novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; + did | cid | dlevel | dauthor | dtitle +-----+-----+--------+-------------------+----------------------- + 78 | 33 | 1 | rls_regress_user1 | some technology novel +(1 row) + +-- Same query a third time, but now fails due to existing tuple finally not +-- passing quals: +INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'some technology novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; +ERROR: new row violates row level security policy (USING expression) for "document" +-- Don't fail just because INSERT doesn't satisfy WITH CHECK option that +-- originated as a barrier/USING() qual from the UPDATE. Note that the UPDATE +-- path *isn't* taken, and so UPDATE-related policy does not apply: +INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'rls_regress_user1', 'technology book, can only insert') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; + did | cid | dlevel | dauthor | dtitle +-----+-----+--------+-------------------+---------------------------------- + 79 | 33 | 1 | rls_regress_user1 | technology book, can only insert +(1 row) + +-- But this time, the same statement fails, because the UPDATE path is taken, +-- and updating the row just inserted falls afoul of security barrier qual +-- (enforced as WCO) -- what we might have updated target tuple to is +-- irrelevant, in fact. +INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'rls_regress_user1', 'technology book, can only insert') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; +ERROR: new row violates row level security policy (USING expression) for "document" +-- Test default USING qual enforced as WCO +SET SESSION AUTHORIZATION rls_regress_user0; +DROP POLICY p1 ON document; +DROP POLICY p2 ON document; +DROP POLICY p3 ON document; +CREATE POLICY p3_with_default ON document FOR UPDATE + USING (cid = (SELECT cid from category WHERE cname = 'novel')); +SET SESSION AUTHORIZATION rls_regress_user1; +-- Just because WCO-style enforcement of USING quals occurs with +-- existing/target tuple does not mean that the implementation can be allowed +-- to fail to also enforce this qual against the final tuple appended to +-- relation (since in the absence of an explicit WCO, this is also interpreted +-- as an UPDATE/ALL WCO in general). +-- +-- UPDATE path is taken here (fails due to existing tuple). Note that this is +-- not reported as a "USING expression", because it's an RLS UPDATE check that originated as +-- a USING qual for the purposes of RLS in general, as opposed to an explicit +-- USING qual that is ordinarily a security barrier. We leave it up to the +-- UPDATE to make this fail: +INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'rls_regress_user1', 'technology book, can only insert') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; +ERROR: new row violates row level security policy for "document" +-- UPDATE path is taken here. Existing tuple passes, since it's cid +-- corresponds to "novel", but default USING qual is enforced against +-- post-UPDATE tuple too (as always when updating with a policy that lacks an +-- explicit WCO), and so this fails: +INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'technology'), 1, 'rls_regress_user1', 'my first novel') + ON CONFLICT (did) DO UPDATE SET cid = EXCLUDED.cid, dtitle = EXCLUDED.dtitle RETURNING *; +ERROR: new row violates row level security policy for "document" +SET SESSION AUTHORIZATION rls_regress_user0; +DROP POLICY p3_with_default ON document; +-- +-- Test ALL policies with ON CONFLICT DO UPDATE (much the same as existing UPDATE +-- tests) +-- +CREATE POLICY p3_with_all ON document FOR ALL + USING (cid = (SELECT cid from category WHERE cname = 'novel')) + WITH CHECK (dauthor = current_user); +SET SESSION AUTHORIZATION rls_regress_user1; +-- Fails, since ALL WCO is enforced in insert path: +INSERT INTO document VALUES (80, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user2', 'my first novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33; +ERROR: new row violates row level security policy for "document" +-- Fails, since ALL policy USING qual is enforced (existing, target tuple is in +-- violation, since it has the "manga" cid): +INSERT INTO document VALUES (4, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'my first novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; +ERROR: new row violates row level security policy (USING expression) for "document" +-- Fails, since ALL WCO are enforced: +INSERT INTO document VALUES (1, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'my first novel') + ON CONFLICT (did) DO UPDATE SET dauthor = 'rls_regress_user2'; +ERROR: new row violates row level security policy for "document" +-- -- ROLE/GROUP -- SET SESSION AUTHORIZATION rls_regress_user0; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index f7f016be219..2df24c0de16 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1123,6 +1123,10 @@ SELECT * FROM shoelace_log ORDER BY sl_name; SELECT * FROM shoelace_obsolete WHERE sl_avail = 0; insert into shoelace values ('sl9', 0, 'pink', 35.0, 'inch', 0.0); insert into shoelace values ('sl10', 1000, 'magenta', 40.0, 'inch', 0.0); +-- Unsupported (even though a similar updatable view construct is) +insert into shoelace values ('sl10', 1000, 'magenta', 40.0, 'inch', 0.0) + on conflict do nothing; +ERROR: INSERT with ON CONFLICT clause cannot be used with table that has INSERT or UPDATE rules SELECT * FROM shoelace_obsolete ORDER BY sl_len_cm; sl_name | sl_avail | sl_color | sl_len | sl_unit | sl_len_cm ------------+----------+------------+--------+----------+----------- @@ -2362,6 +2366,22 @@ DETAIL: Key (id3a, id3c)=(1, 13) is not present in table "rule_and_refint_t2". insert into rule_and_refint_t3 values (1, 13, 11, 'row6'); ERROR: insert or update on table "rule_and_refint_t3" violates foreign key constraint "rule_and_refint_t3_id3a_fkey" DETAIL: Key (id3a, id3b)=(1, 13) is not present in table "rule_and_refint_t1". +-- Ordinary table +insert into rule_and_refint_t3 values (1, 13, 11, 'row6') + on conflict do nothing; +ERROR: insert or update on table "rule_and_refint_t3" violates foreign key constraint "rule_and_refint_t3_id3a_fkey" +DETAIL: Key (id3a, id3b)=(1, 13) is not present in table "rule_and_refint_t1". +-- rule not fired, so fk violation +insert into rule_and_refint_t3 values (1, 13, 11, 'row6') + on conflict (id3a, id3b, id3c) do update + set id3b = excluded.id3b; +ERROR: insert or update on table "rule_and_refint_t3" violates foreign key constraint "rule_and_refint_t3_id3a_fkey" +DETAIL: Key (id3a, id3b)=(1, 13) is not present in table "rule_and_refint_t1". +-- rule fired, so unsupported +insert into shoelace values ('sl9', 0, 'pink', 35.0, 'inch', 0.0) + on conflict (sl_name) do update + set sl_avail = excluded.sl_avail; +ERROR: INSERT with ON CONFLICT clause cannot be used with table that has INSERT or UPDATE rules create rule rule_and_refint_t3_ins as on insert to rule_and_refint_t3 where (exists (select 1 from rule_and_refint_t3 where (((rule_and_refint_t3.id3a = new.id3a) @@ -2743,3 +2763,73 @@ View definition: FROM ( VALUES (1,2)) v(q, w); drop view rule_v1; +-- +-- Check DO INSTEAD rules with ON CONFLICT +-- +CREATE TABLE hats ( + hat_name char(10) primary key, + hat_color char(10) -- hat color +); +CREATE TABLE hat_data ( + hat_name char(10) primary key, + hat_color char(10) -- hat color +); +-- okay +CREATE RULE hat_nosert AS ON INSERT TO hats + DO INSTEAD + INSERT INTO hat_data VALUES ( + NEW.hat_name, + NEW.hat_color) + ON CONFLICT (hat_name) DO NOTHING RETURNING *; +-- Works (projects row) +INSERT INTO hats VALUES ('h7', 'black') RETURNING *; + hat_name | hat_color +------------+------------ + h7 | black +(1 row) + +-- Works (does nothing) +INSERT INTO hats VALUES ('h7', 'black') RETURNING *; + hat_name | hat_color +----------+----------- +(0 rows) + +SELECT tablename, rulename, definition FROM pg_rules + WHERE tablename = 'hats'; + tablename | rulename | definition +-----------+------------+------------------------------------------------------------------------------ + hats | hat_nosert | CREATE RULE hat_nosert AS + + | | ON INSERT TO hats DO INSTEAD INSERT INTO hat_data (hat_name, hat_color)+ + | | VALUES (new.hat_name, new.hat_color) ON CONFLICT DO NOTHING + + | | RETURNING hat_data.hat_name, + + | | hat_data.hat_color; +(1 row) + +DROP RULE hat_nosert ON hats; +CREATE RULE hat_upsert AS ON INSERT TO hats + DO INSTEAD + INSERT INTO hat_data VALUES ( + NEW.hat_name, + NEW.hat_color) + ON CONFLICT (hat_name) DO UPDATE SET hat_color = 'Orange' RETURNING *; +-- Works (does upsert) +INSERT INTO hats VALUES ('h7', 'black') RETURNING *; + hat_name | hat_color +------------+------------ + h7 | Orange +(1 row) + +SELECT tablename, rulename, definition FROM pg_rules + WHERE tablename = 'hats'; + tablename | rulename | definition +-----------+------------+----------------------------------------------------------------------------------------------- + hats | hat_upsert | CREATE RULE hat_upsert AS + + | | ON INSERT TO hats DO INSTEAD INSERT INTO hat_data (hat_name, hat_color) + + | | VALUES (new.hat_name, new.hat_color) ON CONFLICT DO UPDATE SET hat_color = 'Orange'::bpchar+ + | | RETURNING hat_data.hat_name, + + | | hat_data.hat_color; +(1 row) + +DROP RULE hat_upsert ON hats; +drop table hats; +drop table hat_data; diff --git a/src/test/regress/expected/subselect.out b/src/test/regress/expected/subselect.out index b14410fd222..de64ca7ec72 100644 --- a/src/test/regress/expected/subselect.out +++ b/src/test/regress/expected/subselect.out @@ -639,6 +639,28 @@ from (0 rows) -- +-- Test case for subselect within UPDATE of INSERT...ON CONFLICT DO UPDATE +-- +create temp table upsert(key int4 primary key, val text); +insert into upsert values(1, 'val') on conflict (key) do update set val = 'not seen'; +insert into upsert values(1, 'val') on conflict (key) do update set val = 'seen with subselect ' || (select f1 from int4_tbl where f1 != 0 limit 1)::text; +select * from upsert; + key | val +-----+---------------------------- + 1 | seen with subselect 123456 +(1 row) + +with aa as (select 'int4_tbl' u from int4_tbl limit 1) +insert into upsert values (1, 'x'), (999, 'y') +on conflict (key) do update set val = (select u from aa) +returning *; + key | val +-----+---------- + 1 | int4_tbl + 999 | y +(2 rows) + +-- -- Test case for cross-type partial matching in hashed subplan (bug #7597) -- create temp table outer_7597 (f1 int4, f2 int4); diff --git a/src/test/regress/expected/triggers.out b/src/test/regress/expected/triggers.out index f1a5fde107d..3b32e8fdfe3 100644 --- a/src/test/regress/expected/triggers.out +++ b/src/test/regress/expected/triggers.out @@ -274,7 +274,7 @@ drop sequence ttdummy_seq; -- tests for per-statement triggers -- CREATE TABLE log_table (tstamp timestamp default timeofday()::timestamp); -CREATE TABLE main_table (a int, b int); +CREATE TABLE main_table (a int unique, b int); COPY main_table (a,b) FROM stdin; CREATE FUNCTION trigger_func() RETURNS trigger LANGUAGE plpgsql AS ' BEGIN @@ -291,6 +291,14 @@ FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func('after_ins_stmt'); -- CREATE TRIGGER after_upd_stmt_trig AFTER UPDATE ON main_table EXECUTE PROCEDURE trigger_func('after_upd_stmt'); +-- Both insert and update statement level triggers (before and after) should +-- fire. Doesn't fire UPDATE before trigger, but only because one isn't +-- defined. +INSERT INTO main_table (a, b) VALUES (5, 10) ON CONFLICT (a) + DO UPDATE SET b = EXCLUDED.b; +NOTICE: trigger_func(before_ins_stmt) called: action = INSERT, when = BEFORE, level = STATEMENT +NOTICE: trigger_func(after_upd_stmt) called: action = UPDATE, when = AFTER, level = STATEMENT +NOTICE: trigger_func(after_ins_stmt) called: action = INSERT, when = AFTER, level = STATEMENT CREATE TRIGGER after_upd_row_trig AFTER UPDATE ON main_table FOR EACH ROW EXECUTE PROCEDURE trigger_func('after_upd_row'); INSERT INTO main_table DEFAULT VALUES; @@ -305,6 +313,8 @@ NOTICE: trigger_func(after_upd_stmt) called: action = UPDATE, when = AFTER, lev -- UPDATE that effects zero rows should still call per-statement trigger UPDATE main_table SET a = a + 2 WHERE b > 100; NOTICE: trigger_func(after_upd_stmt) called: action = UPDATE, when = AFTER, level = STATEMENT +-- constraint now unneeded +ALTER TABLE main_table DROP CONSTRAINT main_table_a_key; -- COPY should fire per-row and per-statement INSERT triggers COPY main_table (a, b) FROM stdin; NOTICE: trigger_func(before_ins_stmt) called: action = INSERT, when = BEFORE, level = STATEMENT @@ -1731,3 +1741,93 @@ select * from self_ref_trigger; drop table self_ref_trigger; drop function self_ref_trigger_ins_func(); drop function self_ref_trigger_del_func(); +-- +-- Verify behavior of before and after triggers with INSERT...ON CONFLICT +-- DO UPDATE +-- +create table upsert (key int4 primary key, color text); +create function upsert_before_func() + returns trigger language plpgsql as +$$ +begin + if (TG_OP = 'UPDATE') then + raise warning 'before update (old): %', old.*::text; + raise warning 'before update (new): %', new.*::text; + elsif (TG_OP = 'INSERT') then + raise warning 'before insert (new): %', new.*::text; + if new.key % 2 = 0 then + new.key := new.key + 1; + new.color := new.color || ' trig modified'; + raise warning 'before insert (new, modified): %', new.*::text; + end if; + end if; + return new; +end; +$$; +create trigger upsert_before_trig before insert or update on upsert + for each row execute procedure upsert_before_func(); +create function upsert_after_func() + returns trigger language plpgsql as +$$ +begin + if (TG_OP = 'UPDATE') then + raise warning 'after update (old): %', new.*::text; + raise warning 'after update (new): %', new.*::text; + elsif (TG_OP = 'INSERT') then + raise warning 'after insert (new): %', new.*::text; + end if; + return null; +end; +$$; +create trigger upsert_after_trig after insert or update on upsert + for each row execute procedure upsert_after_func(); +insert into upsert values(1, 'black') on conflict (key) do update set color = 'updated ' || upsert.color; +WARNING: before insert (new): (1,black) +WARNING: after insert (new): (1,black) +insert into upsert values(2, 'red') on conflict (key) do update set color = 'updated ' || upsert.color; +WARNING: before insert (new): (2,red) +WARNING: before insert (new, modified): (3,"red trig modified") +WARNING: after insert (new): (3,"red trig modified") +insert into upsert values(3, 'orange') on conflict (key) do update set color = 'updated ' || upsert.color; +WARNING: before insert (new): (3,orange) +WARNING: before update (old): (3,"red trig modified") +WARNING: before update (new): (3,"updated red trig modified") +WARNING: after update (old): (3,"updated red trig modified") +WARNING: after update (new): (3,"updated red trig modified") +insert into upsert values(4, 'green') on conflict (key) do update set color = 'updated ' || upsert.color; +WARNING: before insert (new): (4,green) +WARNING: before insert (new, modified): (5,"green trig modified") +WARNING: after insert (new): (5,"green trig modified") +insert into upsert values(5, 'purple') on conflict (key) do update set color = 'updated ' || upsert.color; +WARNING: before insert (new): (5,purple) +WARNING: before update (old): (5,"green trig modified") +WARNING: before update (new): (5,"updated green trig modified") +WARNING: after update (old): (5,"updated green trig modified") +WARNING: after update (new): (5,"updated green trig modified") +insert into upsert values(6, 'white') on conflict (key) do update set color = 'updated ' || upsert.color; +WARNING: before insert (new): (6,white) +WARNING: before insert (new, modified): (7,"white trig modified") +WARNING: after insert (new): (7,"white trig modified") +insert into upsert values(7, 'pink') on conflict (key) do update set color = 'updated ' || upsert.color; +WARNING: before insert (new): (7,pink) +WARNING: before update (old): (7,"white trig modified") +WARNING: before update (new): (7,"updated white trig modified") +WARNING: after update (old): (7,"updated white trig modified") +WARNING: after update (new): (7,"updated white trig modified") +insert into upsert values(8, 'yellow') on conflict (key) do update set color = 'updated ' || upsert.color; +WARNING: before insert (new): (8,yellow) +WARNING: before insert (new, modified): (9,"yellow trig modified") +WARNING: after insert (new): (9,"yellow trig modified") +select * from upsert; + key | color +-----+----------------------------- + 1 | black + 3 | updated red trig modified + 5 | updated green trig modified + 7 | updated white trig modified + 9 | yellow trig modified +(5 rows) + +drop table upsert; +drop function upsert_before_func(); +drop function upsert_after_func(); diff --git a/src/test/regress/expected/updatable_views.out b/src/test/regress/expected/updatable_views.out index ccabe9e3dcb..7eb92612d73 100644 --- a/src/test/regress/expected/updatable_views.out +++ b/src/test/regress/expected/updatable_views.out @@ -215,6 +215,67 @@ INSERT INTO rw_view15 VALUES (3, 'ROW 3'); -- should fail ERROR: cannot insert into column "upper" of view "rw_view15" DETAIL: View columns that are not columns of their base relation are not updatable. INSERT INTO rw_view15 (a) VALUES (3); -- should be OK +INSERT INTO rw_view15 (a) VALUES (3) ON CONFLICT DO NOTHING; -- succeeds +SELECT * FROM rw_view15; + a | upper +----+------------- + -2 | ROW -2 + -1 | ROW -1 + 0 | ROW 0 + 1 | ROW 1 + 2 | ROW 2 + 3 | UNSPECIFIED +(6 rows) + +INSERT INTO rw_view15 (a) VALUES (3) ON CONFLICT (a) DO NOTHING; -- succeeds +SELECT * FROM rw_view15; + a | upper +----+------------- + -2 | ROW -2 + -1 | ROW -1 + 0 | ROW 0 + 1 | ROW 1 + 2 | ROW 2 + 3 | UNSPECIFIED +(6 rows) + +INSERT INTO rw_view15 (a) VALUES (3) ON CONFLICT (a) DO UPDATE set a = excluded.a; -- succeeds +SELECT * FROM rw_view15; + a | upper +----+------------- + -2 | ROW -2 + -1 | ROW -1 + 0 | ROW 0 + 1 | ROW 1 + 2 | ROW 2 + 3 | UNSPECIFIED +(6 rows) + +INSERT INTO rw_view15 (a) VALUES (3) ON CONFLICT (a) DO UPDATE set upper = 'blarg'; -- fails +ERROR: cannot insert into column "upper" of view "rw_view15" +DETAIL: View columns that are not columns of their base relation are not updatable. +SELECT * FROM rw_view15; + a | upper +----+------------- + -2 | ROW -2 + -1 | ROW -1 + 0 | ROW 0 + 1 | ROW 1 + 2 | ROW 2 + 3 | UNSPECIFIED +(6 rows) + +SELECT * FROM rw_view15; + a | upper +----+------------- + -2 | ROW -2 + -1 | ROW -1 + 0 | ROW 0 + 1 | ROW 1 + 2 | ROW 2 + 3 | UNSPECIFIED +(6 rows) + ALTER VIEW rw_view15 ALTER COLUMN upper SET DEFAULT 'NOT SET'; INSERT INTO rw_view15 (a) VALUES (4); -- should fail ERROR: cannot insert into column "upper" of view "rw_view15" diff --git a/src/test/regress/expected/update.out b/src/test/regress/expected/update.out index 1de2a867a85..adc1fd7c394 100644 --- a/src/test/regress/expected/update.out +++ b/src/test/regress/expected/update.out @@ -6,6 +6,10 @@ CREATE TABLE update_test ( b INT, c TEXT ); +CREATE TABLE upsert_test ( + a INT PRIMARY KEY, + b TEXT +); INSERT INTO update_test VALUES (5, 10, 'foo'); INSERT INTO update_test(b, a) VALUES (15, 10); SELECT * FROM update_test; @@ -147,4 +151,34 @@ SELECT a, b, char_length(c) FROM update_test; 42 | 12 | 10000 (4 rows) +-- Test ON CONFLICT DO UPDATE +INSERT INTO upsert_test VALUES(1, 'Boo'); +-- uncorrelated sub-select: +WITH aaa AS (SELECT 1 AS a, 'Foo' AS b) INSERT INTO upsert_test + VALUES (1, 'Bar') ON CONFLICT(a) + DO UPDATE SET (b, a) = (SELECT b, a FROM aaa) RETURNING *; + a | b +---+----- + 1 | Foo +(1 row) + +-- correlated sub-select: +INSERT INTO upsert_test VALUES (1, 'Baz') ON CONFLICT(a) + DO UPDATE SET (b, a) = (SELECT b || ', Correlated', a from upsert_test i WHERE i.a = upsert_test.a) + RETURNING *; + a | b +---+----------------- + 1 | Foo, Correlated +(1 row) + +-- correlated sub-select (EXCLUDED.* alias): +INSERT INTO upsert_test VALUES (1, 'Bat') ON CONFLICT(a) + DO UPDATE SET (b, a) = (SELECT b || ', Excluded', a from upsert_test i WHERE i.a = excluded.a) + RETURNING *; + a | b +---+--------------------------- + 1 | Foo, Correlated, Excluded +(1 row) + DROP TABLE update_test; +DROP TABLE upsert_test; diff --git a/src/test/regress/expected/with.out b/src/test/regress/expected/with.out index a31ec341e69..2c9226c3db5 100644 --- a/src/test/regress/expected/with.out +++ b/src/test/regress/expected/with.out @@ -1806,6 +1806,88 @@ SELECT * FROM y; -400 (22 rows) +-- data-modifying WITH containing INSERT...ON CONFLICT DO UPDATE +CREATE TABLE z AS SELECT i AS k, (i || ' v')::text v FROM generate_series(1, 16, 3) i; +ALTER TABLE z ADD UNIQUE (k); +WITH t AS ( + INSERT INTO z SELECT i, 'insert' + FROM generate_series(0, 16) i + ON CONFLICT (k) DO UPDATE SET v = z.v || ', now update' + RETURNING * +) +SELECT * FROM t JOIN y ON t.k = y.a ORDER BY a, k; + k | v | a +---+--------+--- + 0 | insert | 0 + 0 | insert | 0 +(2 rows) + +-- Test EXCLUDED.* reference within CTE +WITH aa AS ( + INSERT INTO z VALUES(1, 5) ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v + WHERE z.k != EXCLUDED.k + RETURNING * +) +SELECT * FROM aa; + k | v +---+--- +(0 rows) + +-- New query/snapshot demonstrates side-effects of previous query. +SELECT * FROM z ORDER BY k; + k | v +----+------------------ + 0 | insert + 1 | 1 v, now update + 2 | insert + 3 | insert + 4 | 4 v, now update + 5 | insert + 6 | insert + 7 | 7 v, now update + 8 | insert + 9 | insert + 10 | 10 v, now update + 11 | insert + 12 | insert + 13 | 13 v, now update + 14 | insert + 15 | insert + 16 | 16 v, now update +(17 rows) + +-- +-- Ensure subqueries within the update clause work, even if they +-- reference outside values +-- +WITH aa AS (SELECT 1 a, 2 b) +INSERT INTO z VALUES(1, 'insert') +ON CONFLICT (k) DO UPDATE SET v = (SELECT b || ' update' FROM aa WHERE a = 1 LIMIT 1); +WITH aa AS (SELECT 1 a, 2 b) +INSERT INTO z VALUES(1, 'insert') +ON CONFLICT (k) DO UPDATE SET v = ' update' WHERE z.k = (SELECT a FROM aa); +WITH aa AS (SELECT 1 a, 2 b) +INSERT INTO z VALUES(1, 'insert') +ON CONFLICT (k) DO UPDATE SET v = (SELECT b || ' update' FROM aa WHERE a = 1 LIMIT 1); +WITH aa AS (SELECT 'a' a, 'b' b UNION ALL SELECT 'a' a, 'b' b) +INSERT INTO z VALUES(1, 'insert') +ON CONFLICT (k) DO UPDATE SET v = (SELECT b || ' update' FROM aa WHERE a = 'a' LIMIT 1); +WITH aa AS (SELECT 1 a, 2 b) +INSERT INTO z VALUES(1, (SELECT b || ' insert' FROM aa WHERE a = 1 )) +ON CONFLICT (k) DO UPDATE SET v = (SELECT b || ' update' FROM aa WHERE a = 1 LIMIT 1); +-- This shows an attempt to update an invisible row, which should really be +-- reported as a cardinality violation, but it doesn't seem worth fixing: +WITH simpletup AS ( + SELECT 2 k, 'Green' v), +upsert_cte AS ( + INSERT INTO z VALUES(2, 'Blue') ON CONFLICT (k) DO + UPDATE SET (k, v) = (SELECT k, v FROM simpletup WHERE simpletup.k = z.k) + RETURNING k, v) +INSERT INTO z VALUES(2, 'Red') ON CONFLICT (k) DO +UPDATE SET (k, v) = (SELECT k, v FROM upsert_cte WHERE upsert_cte.k = z.k) +RETURNING k, v; +ERROR: attempted to update invisible tuple +DROP TABLE z; -- check that run to completion happens in proper ordering TRUNCATE TABLE y; INSERT INTO y SELECT generate_series(1, 3); diff --git a/src/test/regress/input/constraints.source b/src/test/regress/input/constraints.source index c16f65088a9..7647544f9b9 100644 --- a/src/test/regress/input/constraints.source +++ b/src/test/regress/input/constraints.source @@ -292,6 +292,11 @@ INSERT INTO UNIQUE_TBL VALUES (5, 'one'); INSERT INTO UNIQUE_TBL (t) VALUES ('six'); INSERT INTO UNIQUE_TBL (t) VALUES ('seven'); +INSERT INTO UNIQUE_TBL VALUES (5, 'five-upsert-insert') ON CONFLICT (i) DO UPDATE SET t = 'five-upsert-update'; +INSERT INTO UNIQUE_TBL VALUES (6, 'six-upsert-insert') ON CONFLICT (i) DO UPDATE SET t = 'six-upsert-update'; +-- should fail +INSERT INTO UNIQUE_TBL VALUES (1, 'a'), (2, 'b'), (2, 'b') ON CONFLICT (i) DO UPDATE SET t = 'fails'; + SELECT '' AS five, * FROM UNIQUE_TBL; DROP TABLE UNIQUE_TBL; @@ -438,6 +443,12 @@ INSERT INTO circles VALUES('<(0,0), 5>', '<(0,0), 4>'); INSERT INTO circles VALUES('<(10,10), 10>', '<(0,0), 5>'); -- fail, overlaps INSERT INTO circles VALUES('<(20,20), 10>', '<(0,0), 4>'); +-- succeed, because violation is ignored +INSERT INTO circles VALUES('<(20,20), 10>', '<(0,0), 4>') + ON CONFLICT ON CONSTRAINT circles_c1_c2_excl DO NOTHING; +-- fail, because DO UPDATE variant requires unique index +INSERT INTO circles VALUES('<(20,20), 10>', '<(0,0), 4>') + ON CONFLICT ON CONSTRAINT circles_c1_c2_excl DO UPDATE SET c2 = EXCLUDED.c2; -- succeed because c1 doesn't overlap INSERT INTO circles VALUES('<(20,20), 1>', '<(0,0), 5>'); -- succeed because c2 doesn't overlap @@ -462,6 +473,7 @@ CREATE TABLE deferred_excl ( INSERT INTO deferred_excl VALUES(1); INSERT INTO deferred_excl VALUES(2); INSERT INTO deferred_excl VALUES(1); -- fail +INSERT INTO deferred_excl VALUES(1) ON CONFLICT ON CONSTRAINT deferred_excl_con DO NOTHING; -- fail BEGIN; INSERT INTO deferred_excl VALUES(2); -- no fail here COMMIT; -- should fail here diff --git a/src/test/regress/output/constraints.source b/src/test/regress/output/constraints.source index d3ec2333139..bbe4ed19760 100644 --- a/src/test/regress/output/constraints.source +++ b/src/test/regress/output/constraints.source @@ -421,16 +421,23 @@ INSERT INTO UNIQUE_TBL VALUES (4, 'four'); INSERT INTO UNIQUE_TBL VALUES (5, 'one'); INSERT INTO UNIQUE_TBL (t) VALUES ('six'); INSERT INTO UNIQUE_TBL (t) VALUES ('seven'); +INSERT INTO UNIQUE_TBL VALUES (5, 'five-upsert-insert') ON CONFLICT (i) DO UPDATE SET t = 'five-upsert-update'; +INSERT INTO UNIQUE_TBL VALUES (6, 'six-upsert-insert') ON CONFLICT (i) DO UPDATE SET t = 'six-upsert-update'; +-- should fail +INSERT INTO UNIQUE_TBL VALUES (1, 'a'), (2, 'b'), (2, 'b') ON CONFLICT (i) DO UPDATE SET t = 'fails'; +ERROR: ON CONFLICT DO UPDATE command cannot affect row a second time +HINT: Ensure that no rows proposed for insertion within the same command have duplicate constrained values. SELECT '' AS five, * FROM UNIQUE_TBL; - five | i | t -------+---+------- + five | i | t +------+---+-------------------- | 1 | one | 2 | two | 4 | four - | 5 | one | | six | | seven -(6 rows) + | 5 | five-upsert-update + | 6 | six-upsert-insert +(7 rows) DROP TABLE UNIQUE_TBL; CREATE TABLE UNIQUE_TBL (i int, t text, @@ -605,6 +612,13 @@ INSERT INTO circles VALUES('<(10,10), 10>', '<(0,0), 5>'); INSERT INTO circles VALUES('<(20,20), 10>', '<(0,0), 4>'); ERROR: conflicting key value violates exclusion constraint "circles_c1_c2_excl" DETAIL: Key (c1, (c2::circle))=(<(20,20),10>, <(0,0),4>) conflicts with existing key (c1, (c2::circle))=(<(10,10),10>, <(0,0),5>). +-- succeed, because violation is ignored +INSERT INTO circles VALUES('<(20,20), 10>', '<(0,0), 4>') + ON CONFLICT ON CONSTRAINT circles_c1_c2_excl DO NOTHING; +-- fail, because DO UPDATE variant requires unique index +INSERT INTO circles VALUES('<(20,20), 10>', '<(0,0), 4>') + ON CONFLICT ON CONSTRAINT circles_c1_c2_excl DO UPDATE SET c2 = EXCLUDED.c2; +ERROR: ON CONFLICT DO UPDATE not supported with exclusion constraints -- succeed because c1 doesn't overlap INSERT INTO circles VALUES('<(20,20), 1>', '<(0,0), 5>'); -- succeed because c2 doesn't overlap @@ -627,6 +641,8 @@ INSERT INTO deferred_excl VALUES(2); INSERT INTO deferred_excl VALUES(1); -- fail ERROR: conflicting key value violates exclusion constraint "deferred_excl_con" DETAIL: Key (f1)=(1) conflicts with existing key (f1)=(1). +INSERT INTO deferred_excl VALUES(1) ON CONFLICT ON CONSTRAINT deferred_excl_con DO NOTHING; -- fail +ERROR: ON CONFLICT does not support deferred unique constraints/exclusion constraints as arbiters BEGIN; INSERT INTO deferred_excl VALUES(2); -- no fail here COMMIT; -- should fail here diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 6d3b865351d..b0ebb6b3f4c 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -36,6 +36,7 @@ test: geometry horology regex oidjoins type_sanity opr_sanity # These four each depend on the previous one # ---------- test: insert +test: insert_conflict test: create_function_1 test: create_type test: create_table diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 8326894ed9d..8409c0f3ef2 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -50,6 +50,7 @@ test: oidjoins test: type_sanity test: opr_sanity test: insert +test: insert_conflict test: create_function_1 test: create_type test: create_table diff --git a/src/test/regress/sql/insert_conflict.sql b/src/test/regress/sql/insert_conflict.sql new file mode 100644 index 00000000000..ba2b66bdb67 --- /dev/null +++ b/src/test/regress/sql/insert_conflict.sql @@ -0,0 +1,284 @@ +-- +-- insert...on conflict do unique index inference +-- +create table insertconflicttest(key int4, fruit text); + +-- +-- Test unique index inference with operator class specifications and +-- named collations +-- +create unique index op_index_key on insertconflicttest(key, fruit text_pattern_ops); +create unique index collation_index_key on insertconflicttest(key, fruit collate "C"); +create unique index both_index_key on insertconflicttest(key, fruit collate "C" text_pattern_ops); +create unique index both_index_expr_key on insertconflicttest(key, lower(fruit) collate "C" text_pattern_ops); + +-- fails +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (key) do nothing; +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (fruit) do nothing; + +-- succeeds +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (key, fruit) do nothing; +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (fruit, key, fruit, key) do nothing; +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (lower(fruit), key, lower(fruit), key) do nothing; +-- Neither collation nor operator class specifications are required -- +-- supplying them merely *limits* matches to indexes with matching opclasses +-- used for relevant indexes +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (key, fruit text_pattern_ops) do nothing; +-- Okay, arbitrates using both index where text_pattern_ops opclass does and +-- does not appear. +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (key, fruit collate "C") do nothing; +-- Okay, but only accepts the single index where both opclass and collation are +-- specified +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (fruit collate "C" text_pattern_ops, key) do nothing; +-- Okay, but only accepts the single index where both opclass and collation are +-- specified (plus expression variant) +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (lower(fruit) collate "C", key, key) do nothing; +-- Attribute appears twice, while not all attributes/expressions on attributes +-- appearing within index definition match in terms of both opclass and +-- collation. +-- +-- Works because every attribute in inference specification needs to be +-- satisfied once or more by cataloged index attribute, and as always when an +-- attribute in the cataloged definition has a non-default opclass/collation, +-- it still satisfied some inference attribute lacking any particular +-- opclass/collation specification. +-- +-- The implementation is liberal in accepting inference specifications on the +-- assumption that multiple inferred unique indexes will prevent problematic +-- cases. It rolls with unique indexes where attributes redundantly appear +-- multiple times, too (which is not tested here). +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (fruit, key, fruit text_pattern_ops, key) do nothing; +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (lower(fruit) collate "C" text_pattern_ops, key, key) do nothing; + +drop index op_index_key; +drop index collation_index_key; +drop index both_index_key; +drop index both_index_expr_key; + +-- +-- Single key tests +-- +create unique index key_index on insertconflicttest(key); + +-- +-- Explain tests +-- +explain (costs off) insert into insertconflicttest values (0, 'Bilberry') on conflict (key) do update set fruit = excluded.fruit; +-- Should display qual actually attributable to internal sequential scan: +explain (costs off) insert into insertconflicttest values (0, 'Bilberry') on conflict (key) do update set fruit = excluded.fruit where insertconflicttest.fruit != 'Cawesh'; +-- With EXCLUDED.* expression in scan node: +explain (costs off) insert into insertconflicttest values(0, 'Crowberry') on conflict (key) do update set fruit = excluded.fruit where excluded.fruit != 'Elderberry'; +-- Does the same, but JSON format shows "Conflict Arbiter Index" as JSON array: +explain (costs off, format json) insert into insertconflicttest values (0, 'Bilberry') on conflict (key) do update set fruit = excluded.fruit where insertconflicttest.fruit != 'Lime' returning *; + +-- Fails (no unique index inference specification, required for do update variant): +insert into insertconflicttest values (1, 'Apple') on conflict do update set fruit = excluded.fruit; + +-- inference succeeds: +insert into insertconflicttest values (1, 'Apple') on conflict (key) do update set fruit = excluded.fruit; +insert into insertconflicttest values (2, 'Orange') on conflict (key, key, key) do update set fruit = excluded.fruit; + +-- Succeed, since multi-assignment does not involve subquery: +insert into insertconflicttest +values (1, 'Apple'), (2, 'Orange') +on conflict (key) do update set (fruit, key) = (excluded.fruit, excluded.key); + +-- Give good diagnostic message when EXCLUDED.* spuriously referenced from +-- RETURNING: +insert into insertconflicttest values (1, 'Apple') on conflict (key) do update set fruit = excluded.fruit RETURNING excluded.fruit; + +-- Only suggest <table>.* column when inference element misspelled: +insert into insertconflicttest values (1, 'Apple') on conflict (keyy) do update set fruit = excluded.fruit; + +-- Have useful HINT for EXCLUDED.* RTE within UPDATE: +insert into insertconflicttest values (1, 'Apple') on conflict (key) do update set fruit = excluded.fruitt; + +-- inference fails: +insert into insertconflicttest values (3, 'Kiwi') on conflict (key, fruit) do update set fruit = excluded.fruit; +insert into insertconflicttest values (4, 'Mango') on conflict (fruit, key) do update set fruit = excluded.fruit; +insert into insertconflicttest values (5, 'Lemon') on conflict (fruit) do update set fruit = excluded.fruit; +insert into insertconflicttest values (6, 'Passionfruit') on conflict (lower(fruit)) do update set fruit = excluded.fruit; + +-- Check the target relation can be aliased +insert into insertconflicttest values (6, 'Passionfruits') on conflict (key) do update set fruit = excluded.fruit; +insert into insertconflicttest AS ict values (6, 'Passionfruit') on conflict (key) do update set fruit = excluded.fruit; -- ok, no reference to target table +insert into insertconflicttest AS ict values (6, 'Passionfruit') on conflict (key) do update set fruit = ict.fruit; -- ok, alias +insert into insertconflicttest AS ict values (6, 'Passionfruit') on conflict (key) do update set fruit = insertconflicttest.fruit; -- error, references aliased away name + +drop index key_index; + +-- +-- Composite key tests +-- +create unique index comp_key_index on insertconflicttest(key, fruit); + +-- inference succeeds: +insert into insertconflicttest values (7, 'Raspberry') on conflict (key, fruit) do update set fruit = excluded.fruit; +insert into insertconflicttest values (8, 'Lime') on conflict (fruit, key) do update set fruit = excluded.fruit; + +-- inference fails: +insert into insertconflicttest values (9, 'Banana') on conflict (key) do update set fruit = excluded.fruit; +insert into insertconflicttest values (10, 'Blueberry') on conflict (key, key, key) do update set fruit = excluded.fruit; +insert into insertconflicttest values (11, 'Cherry') on conflict (key, lower(fruit)) do update set fruit = excluded.fruit; +insert into insertconflicttest values (12, 'Date') on conflict (lower(fruit), key) do update set fruit = excluded.fruit; + +drop index comp_key_index; + +-- +-- Partial index tests, no inference predicate specificied +-- +create unique index part_comp_key_index on insertconflicttest(key, fruit) where key < 5; +create unique index expr_part_comp_key_index on insertconflicttest(key, lower(fruit)) where key < 5; + +-- inference fails: +insert into insertconflicttest values (13, 'Grape') on conflict (key, fruit) do update set fruit = excluded.fruit; +insert into insertconflicttest values (14, 'Raisin') on conflict (fruit, key) do update set fruit = excluded.fruit; +insert into insertconflicttest values (15, 'Cranberry') on conflict (key) do update set fruit = excluded.fruit; +insert into insertconflicttest values (16, 'Melon') on conflict (key, key, key) do update set fruit = excluded.fruit; +insert into insertconflicttest values (17, 'Mulberry') on conflict (key, lower(fruit)) do update set fruit = excluded.fruit; +insert into insertconflicttest values (18, 'Pineapple') on conflict (lower(fruit), key) do update set fruit = excluded.fruit; + +drop index part_comp_key_index; +drop index expr_part_comp_key_index; + +-- +-- Expression index tests +-- +create unique index expr_key_index on insertconflicttest(lower(fruit)); + +-- inference succeeds: +insert into insertconflicttest values (20, 'Quince') on conflict (lower(fruit)) do update set fruit = excluded.fruit; +insert into insertconflicttest values (21, 'Pomegranate') on conflict (lower(fruit), lower(fruit)) do update set fruit = excluded.fruit; + +-- inference fails: +insert into insertconflicttest values (22, 'Apricot') on conflict (upper(fruit)) do update set fruit = excluded.fruit; +insert into insertconflicttest values (23, 'Blackberry') on conflict (fruit) do update set fruit = excluded.fruit; + +drop index expr_key_index; + +-- +-- Expression index tests (with regular column) +-- +create unique index expr_comp_key_index on insertconflicttest(key, lower(fruit)); +create unique index tricky_expr_comp_key_index on insertconflicttest(key, lower(fruit), upper(fruit)); + +-- inference succeeds: +insert into insertconflicttest values (24, 'Plum') on conflict (key, lower(fruit)) do update set fruit = excluded.fruit; +insert into insertconflicttest values (25, 'Peach') on conflict (lower(fruit), key) do update set fruit = excluded.fruit; +-- Should not infer "tricky_expr_comp_key_index" index: +explain (costs off) insert into insertconflicttest values (26, 'Fig') on conflict (lower(fruit), key, lower(fruit), key) do update set fruit = excluded.fruit; + +-- inference fails: +insert into insertconflicttest values (27, 'Prune') on conflict (key, upper(fruit)) do update set fruit = excluded.fruit; +insert into insertconflicttest values (28, 'Redcurrant') on conflict (fruit, key) do update set fruit = excluded.fruit; +insert into insertconflicttest values (29, 'Nectarine') on conflict (key) do update set fruit = excluded.fruit; + +drop index expr_comp_key_index; +drop index tricky_expr_comp_key_index; + +-- +-- Non-spurious duplicate violation tests +-- +create unique index key_index on insertconflicttest(key); +create unique index fruit_index on insertconflicttest(fruit); + +-- succeeds, since UPDATE happens to update "fruit" to existing value: +insert into insertconflicttest values (26, 'Fig') on conflict (key) do update set fruit = excluded.fruit; +-- fails, since UPDATE is to row with key value 26, and we're updating "fruit" +-- to a value that happens to exist in another row ('peach'): +insert into insertconflicttest values (26, 'Peach') on conflict (key) do update set fruit = excluded.fruit; +-- succeeds, since "key" isn't repeated/referenced in UPDATE, and "fruit" +-- arbitrates that statement updates existing "Fig" row: +insert into insertconflicttest values (25, 'Fig') on conflict (fruit) do update set fruit = excluded.fruit; + +drop index key_index; +drop index fruit_index; + +-- +-- Test partial unique index inference +-- +create unique index partial_key_index on insertconflicttest(key) where fruit like '%berry'; + +-- Succeeds +insert into insertconflicttest values (23, 'Blackberry') on conflict (key) where fruit like '%berry' do update set fruit = excluded.fruit; +insert into insertconflicttest values (23, 'Blackberry') on conflict (key) where fruit like '%berry' and fruit = 'inconsequential' do nothing; + +-- fails +insert into insertconflicttest values (23, 'Blackberry') on conflict (key) do update set fruit = excluded.fruit; +insert into insertconflicttest values (23, 'Blackberry') on conflict (key) where fruit like '%berry' or fruit = 'consequential' do nothing; +insert into insertconflicttest values (23, 'Blackberry') on conflict (fruit) where fruit like '%berry' do update set fruit = excluded.fruit; + +drop index partial_key_index; + +-- Cleanup +drop table insertconflicttest; + +-- ****************************************************************** +-- * * +-- * Test inheritance (example taken from tutorial) * +-- * * +-- ****************************************************************** +create table cities ( + name text, + population float8, + altitude int -- (in ft) +); + +create table capitals ( + state char(2) +) inherits (cities); + +-- Create unique indexes. Due to a general limitation of inheritance, +-- uniqueness is only enforced per-relation. Unique index inference +-- specification will do the right thing, though. +create unique index cities_names_unique on cities (name); +create unique index capitals_names_unique on capitals (name); + +-- prepopulate the tables. +insert into cities values ('San Francisco', 7.24E+5, 63); +insert into cities values ('Las Vegas', 2.583E+5, 2174); +insert into cities values ('Mariposa', 1200, 1953); + +insert into capitals values ('Sacramento', 3.694E+5, 30, 'CA'); +insert into capitals values ('Madison', 1.913E+5, 845, 'WI'); + +-- Tests proper for inheritance: +select * from capitals; + +-- Succeeds: +insert into cities values ('Las Vegas', 2.583E+5, 2174) on conflict do nothing; +insert into capitals values ('Sacramento', 4664.E+5, 30, 'CA') on conflict (name) do update set population = excluded.population; +-- Wrong "Sacramento", so do nothing: +insert into capitals values ('Sacramento', 50, 2267, 'NE') on conflict (name) do nothing; +select * from capitals; +insert into cities values ('Las Vegas', 5.83E+5, 2001) on conflict (name) do update set population = excluded.population, altitude = excluded.altitude; +select tableoid::regclass, * from cities; +insert into capitals values ('Las Vegas', 5.83E+5, 2222, 'NV') on conflict (name) do update set population = excluded.population; +-- Capitals will contain new capital, Las Vegas: +select * from capitals; +-- Cities contains two instances of "Las Vegas", since unique constraints don't +-- work across inheritance: +select tableoid::regclass, * from cities; +-- This only affects "cities" version of "Las Vegas": +insert into cities values ('Las Vegas', 5.86E+5, 2223) on conflict (name) do update set population = excluded.population, altitude = excluded.altitude; +select tableoid::regclass, * from cities; + +-- clean up +drop table capitals; +drop table cities; + + +-- Make sure a table named excluded is handled properly +create table excluded(key int primary key, data text); +insert into excluded values(1, '1'); +-- error, ambiguous +insert into excluded values(1, '2') on conflict (key) do update set data = excluded.data RETURNING *; +-- ok, aliased +insert into excluded AS target values(1, '2') on conflict (key) do update set data = excluded.data RETURNING *; +-- ok, aliased +insert into excluded AS target values(1, '2') on conflict (key) do update set data = target.data RETURNING *; +-- make sure excluded isn't a problem in returning clause +insert into excluded values(1, '2') on conflict (key) do update set data = 3 RETURNING excluded.*; + +-- clean up +drop table excluded; diff --git a/src/test/regress/sql/privileges.sql b/src/test/regress/sql/privileges.sql index f97a75a5fdc..22b54a28c46 100644 --- a/src/test/regress/sql/privileges.sql +++ b/src/test/regress/sql/privileges.sql @@ -194,7 +194,7 @@ SELECT * FROM atestv2; -- fail (even though regressuser2 can access underlying a -- Test column level permissions SET SESSION AUTHORIZATION regressuser1; -CREATE TABLE atest5 (one int, two int, three int); +CREATE TABLE atest5 (one int, two int unique, three int, four int unique); CREATE TABLE atest6 (one int, two int, blue int); GRANT SELECT (one), INSERT (two), UPDATE (three) ON atest5 TO regressuser4; GRANT ALL (one) ON atest5 TO regressuser3; @@ -245,6 +245,23 @@ INSERT INTO atest5 VALUES (5,5,5); -- fail UPDATE atest5 SET three = 10; -- ok UPDATE atest5 SET one = 8; -- fail UPDATE atest5 SET three = 5, one = 2; -- fail +-- Check that column level privs are enforced in RETURNING +-- Ok. +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = 10; +-- Error. No SELECT on column three. +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = 10 RETURNING atest5.three; +-- Ok. May SELECT on column "one": +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = 10 RETURNING atest5.one; +-- Check that column level privileges are enforced for EXCLUDED +-- Ok. we may select one +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = EXCLUDED.one; +-- Error. No select rights on three +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = EXCLUDED.three; +INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set one = 8; -- fails (due to UPDATE) +INSERT INTO atest5(three) VALUES (4) ON CONFLICT (two) DO UPDATE set three = 10; -- fails (due to INSERT) +-- Check that the the columns in the inference require select privileges +-- Error. No privs on four +INSERT INTO atest5(three) VALUES (4) ON CONFLICT (four) DO UPDATE set three = 10; SET SESSION AUTHORIZATION regressuser1; REVOKE ALL (one) ON atest5 FROM regressuser4; diff --git a/src/test/regress/sql/returning.sql b/src/test/regress/sql/returning.sql index 0ed9a489510..a460f82fb7c 100644 --- a/src/test/regress/sql/returning.sql +++ b/src/test/regress/sql/returning.sql @@ -154,3 +154,9 @@ UPDATE joinview SET f1 = f1 + 1 WHERE f3 = 57 RETURNING *, other + 1; SELECT * FROM joinview; SELECT * FROM foo; SELECT * FROM voo; + +-- Check aliased target relation +INSERT INTO foo AS bar DEFAULT VALUES RETURNING *; -- ok +INSERT INTO foo AS bar DEFAULT VALUES RETURNING foo.*; -- fails, wrong name +INSERT INTO foo AS bar DEFAULT VALUES RETURNING bar.*; -- ok +INSERT INTO foo AS bar DEFAULT VALUES RETURNING bar.f3; -- ok diff --git a/src/test/regress/sql/rowsecurity.sql b/src/test/regress/sql/rowsecurity.sql index 7d12dd00a2f..e8c5932b201 100644 --- a/src/test/regress/sql/rowsecurity.sql +++ b/src/test/regress/sql/rowsecurity.sql @@ -511,6 +511,118 @@ DELETE FROM bv1 WHERE a = 6 AND f_leak(b); SET SESSION AUTHORIZATION rls_regress_user0; SELECT * FROM b1; +-- +-- INSERT ... ON CONFLICT DO UPDATE and Row-level security +-- + +SET SESSION AUTHORIZATION rls_regress_user0; +DROP POLICY p1 ON document; + +CREATE POLICY p1 ON document FOR SELECT USING (true); +CREATE POLICY p2 ON document FOR INSERT WITH CHECK (dauthor = current_user); +CREATE POLICY p3 ON document FOR UPDATE + USING (cid = (SELECT cid from category WHERE cname = 'novel')) + WITH CHECK (dauthor = current_user); + +SET SESSION AUTHORIZATION rls_regress_user1; + +-- Exists... +SELECT * FROM document WHERE did = 2; + +-- ...so violates actual WITH CHECK OPTION within UPDATE (not INSERT, since +-- alternative UPDATE path happens to be taken): +INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user2', 'my first novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, dauthor = EXCLUDED.dauthor; + +-- Violates USING qual for UPDATE policy p3. +-- +-- UPDATE path is taken, but UPDATE fails purely because *existing* row to be +-- updated is not a "novel"/cid 11 (row is not leaked, even though we have +-- SELECT privileges sufficient to see the row in this instance): +INSERT INTO document VALUES (33, 22, 1, 'rls_regress_user1', 'okay science fiction'); -- preparation for next statement +INSERT INTO document VALUES (33, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'Some novel, replaces sci-fi') -- takes UPDATE path + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; +-- Fine (we UPDATE, since INSERT WCOs and UPDATE security barrier quals + WCOs +-- not violated): +INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'my first novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; +-- Fine (we INSERT, so "cid = 33" ("technology") isn't evaluated): +INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'some technology novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; +-- Fine (same query, but we UPDATE, so "cid = 33", ("technology") is not the +-- case in respect of *existing* tuple): +INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'some technology novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; +-- Same query a third time, but now fails due to existing tuple finally not +-- passing quals: +INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'some technology novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; +-- Don't fail just because INSERT doesn't satisfy WITH CHECK option that +-- originated as a barrier/USING() qual from the UPDATE. Note that the UPDATE +-- path *isn't* taken, and so UPDATE-related policy does not apply: +INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'rls_regress_user1', 'technology book, can only insert') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; +-- But this time, the same statement fails, because the UPDATE path is taken, +-- and updating the row just inserted falls afoul of security barrier qual +-- (enforced as WCO) -- what we might have updated target tuple to is +-- irrelevant, in fact. +INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'rls_regress_user1', 'technology book, can only insert') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; + +-- Test default USING qual enforced as WCO +SET SESSION AUTHORIZATION rls_regress_user0; +DROP POLICY p1 ON document; +DROP POLICY p2 ON document; +DROP POLICY p3 ON document; + +CREATE POLICY p3_with_default ON document FOR UPDATE + USING (cid = (SELECT cid from category WHERE cname = 'novel')); + +SET SESSION AUTHORIZATION rls_regress_user1; +-- Just because WCO-style enforcement of USING quals occurs with +-- existing/target tuple does not mean that the implementation can be allowed +-- to fail to also enforce this qual against the final tuple appended to +-- relation (since in the absence of an explicit WCO, this is also interpreted +-- as an UPDATE/ALL WCO in general). +-- +-- UPDATE path is taken here (fails due to existing tuple). Note that this is +-- not reported as a "USING expression", because it's an RLS UPDATE check that originated as +-- a USING qual for the purposes of RLS in general, as opposed to an explicit +-- USING qual that is ordinarily a security barrier. We leave it up to the +-- UPDATE to make this fail: +INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'rls_regress_user1', 'technology book, can only insert') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; + +-- UPDATE path is taken here. Existing tuple passes, since it's cid +-- corresponds to "novel", but default USING qual is enforced against +-- post-UPDATE tuple too (as always when updating with a policy that lacks an +-- explicit WCO), and so this fails: +INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'technology'), 1, 'rls_regress_user1', 'my first novel') + ON CONFLICT (did) DO UPDATE SET cid = EXCLUDED.cid, dtitle = EXCLUDED.dtitle RETURNING *; + +SET SESSION AUTHORIZATION rls_regress_user0; +DROP POLICY p3_with_default ON document; + +-- +-- Test ALL policies with ON CONFLICT DO UPDATE (much the same as existing UPDATE +-- tests) +-- +CREATE POLICY p3_with_all ON document FOR ALL + USING (cid = (SELECT cid from category WHERE cname = 'novel')) + WITH CHECK (dauthor = current_user); + +SET SESSION AUTHORIZATION rls_regress_user1; + +-- Fails, since ALL WCO is enforced in insert path: +INSERT INTO document VALUES (80, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user2', 'my first novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33; +-- Fails, since ALL policy USING qual is enforced (existing, target tuple is in +-- violation, since it has the "manga" cid): +INSERT INTO document VALUES (4, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'my first novel') + ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; +-- Fails, since ALL WCO are enforced: +INSERT INTO document VALUES (1, (SELECT cid from category WHERE cname = 'novel'), 1, 'rls_regress_user1', 'my first novel') + ON CONFLICT (did) DO UPDATE SET dauthor = 'rls_regress_user2'; -- -- ROLE/GROUP diff --git a/src/test/regress/sql/rules.sql b/src/test/regress/sql/rules.sql index c385e414578..6f1a1b84e79 100644 --- a/src/test/regress/sql/rules.sql +++ b/src/test/regress/sql/rules.sql @@ -680,6 +680,9 @@ SELECT * FROM shoelace_log ORDER BY sl_name; insert into shoelace values ('sl9', 0, 'pink', 35.0, 'inch', 0.0); insert into shoelace values ('sl10', 1000, 'magenta', 40.0, 'inch', 0.0); +-- Unsupported (even though a similar updatable view construct is) +insert into shoelace values ('sl10', 1000, 'magenta', 40.0, 'inch', 0.0) + on conflict do nothing; SELECT * FROM shoelace_obsolete ORDER BY sl_len_cm; SELECT * FROM shoelace_candelete; @@ -844,6 +847,17 @@ insert into rule_and_refint_t3 values (1, 12, 11, 'row3'); insert into rule_and_refint_t3 values (1, 12, 12, 'row4'); insert into rule_and_refint_t3 values (1, 11, 13, 'row5'); insert into rule_and_refint_t3 values (1, 13, 11, 'row6'); +-- Ordinary table +insert into rule_and_refint_t3 values (1, 13, 11, 'row6') + on conflict do nothing; +-- rule not fired, so fk violation +insert into rule_and_refint_t3 values (1, 13, 11, 'row6') + on conflict (id3a, id3b, id3c) do update + set id3b = excluded.id3b; +-- rule fired, so unsupported +insert into shoelace values ('sl9', 0, 'pink', 35.0, 'inch', 0.0) + on conflict (sl_name) do update + set sl_avail = excluded.sl_avail; create rule rule_and_refint_t3_ins as on insert to rule_and_refint_t3 where (exists (select 1 from rule_and_refint_t3 @@ -1025,3 +1039,48 @@ drop view rule_v1; create view rule_v1(x) as select * from (values(1,2)) v(q,w); \d+ rule_v1 drop view rule_v1; + +-- +-- Check DO INSTEAD rules with ON CONFLICT +-- +CREATE TABLE hats ( + hat_name char(10) primary key, + hat_color char(10) -- hat color +); + +CREATE TABLE hat_data ( + hat_name char(10) primary key, + hat_color char(10) -- hat color +); + +-- okay +CREATE RULE hat_nosert AS ON INSERT TO hats + DO INSTEAD + INSERT INTO hat_data VALUES ( + NEW.hat_name, + NEW.hat_color) + ON CONFLICT (hat_name) DO NOTHING RETURNING *; + +-- Works (projects row) +INSERT INTO hats VALUES ('h7', 'black') RETURNING *; +-- Works (does nothing) +INSERT INTO hats VALUES ('h7', 'black') RETURNING *; +SELECT tablename, rulename, definition FROM pg_rules + WHERE tablename = 'hats'; +DROP RULE hat_nosert ON hats; + +CREATE RULE hat_upsert AS ON INSERT TO hats + DO INSTEAD + INSERT INTO hat_data VALUES ( + NEW.hat_name, + NEW.hat_color) + ON CONFLICT (hat_name) DO UPDATE SET hat_color = 'Orange' RETURNING *; + +-- Works (does upsert) +INSERT INTO hats VALUES ('h7', 'black') RETURNING *; +SELECT tablename, rulename, definition FROM pg_rules + WHERE tablename = 'hats'; +DROP RULE hat_upsert ON hats; + +drop table hats; +drop table hat_data; diff --git a/src/test/regress/sql/subselect.sql b/src/test/regress/sql/subselect.sql index 4be2e40a000..29912230891 100644 --- a/src/test/regress/sql/subselect.sql +++ b/src/test/regress/sql/subselect.sql @@ -374,6 +374,20 @@ from int4_tbl i4 on dummy = i4.f1; -- +-- Test case for subselect within UPDATE of INSERT...ON CONFLICT DO UPDATE +-- +create temp table upsert(key int4 primary key, val text); +insert into upsert values(1, 'val') on conflict (key) do update set val = 'not seen'; +insert into upsert values(1, 'val') on conflict (key) do update set val = 'seen with subselect ' || (select f1 from int4_tbl where f1 != 0 limit 1)::text; + +select * from upsert; + +with aa as (select 'int4_tbl' u from int4_tbl limit 1) +insert into upsert values (1, 'x'), (999, 'y') +on conflict (key) do update set val = (select u from aa) +returning *; + +-- -- Test case for cross-type partial matching in hashed subplan (bug #7597) -- diff --git a/src/test/regress/sql/triggers.sql b/src/test/regress/sql/triggers.sql index 0ea2c314dee..9f66702ceef 100644 --- a/src/test/regress/sql/triggers.sql +++ b/src/test/regress/sql/triggers.sql @@ -208,7 +208,7 @@ drop sequence ttdummy_seq; CREATE TABLE log_table (tstamp timestamp default timeofday()::timestamp); -CREATE TABLE main_table (a int, b int); +CREATE TABLE main_table (a int unique, b int); COPY main_table (a,b) FROM stdin; 5 10 @@ -237,6 +237,12 @@ FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func('after_ins_stmt'); CREATE TRIGGER after_upd_stmt_trig AFTER UPDATE ON main_table EXECUTE PROCEDURE trigger_func('after_upd_stmt'); +-- Both insert and update statement level triggers (before and after) should +-- fire. Doesn't fire UPDATE before trigger, but only because one isn't +-- defined. +INSERT INTO main_table (a, b) VALUES (5, 10) ON CONFLICT (a) + DO UPDATE SET b = EXCLUDED.b; + CREATE TRIGGER after_upd_row_trig AFTER UPDATE ON main_table FOR EACH ROW EXECUTE PROCEDURE trigger_func('after_upd_row'); @@ -246,6 +252,9 @@ UPDATE main_table SET a = a + 1 WHERE b < 30; -- UPDATE that effects zero rows should still call per-statement trigger UPDATE main_table SET a = a + 2 WHERE b > 100; +-- constraint now unneeded +ALTER TABLE main_table DROP CONSTRAINT main_table_a_key; + -- COPY should fire per-row and per-statement INSERT triggers COPY main_table (a, b) FROM stdin; 30 40 @@ -1173,3 +1182,61 @@ select * from self_ref_trigger; drop table self_ref_trigger; drop function self_ref_trigger_ins_func(); drop function self_ref_trigger_del_func(); + +-- +-- Verify behavior of before and after triggers with INSERT...ON CONFLICT +-- DO UPDATE +-- +create table upsert (key int4 primary key, color text); + +create function upsert_before_func() + returns trigger language plpgsql as +$$ +begin + if (TG_OP = 'UPDATE') then + raise warning 'before update (old): %', old.*::text; + raise warning 'before update (new): %', new.*::text; + elsif (TG_OP = 'INSERT') then + raise warning 'before insert (new): %', new.*::text; + if new.key % 2 = 0 then + new.key := new.key + 1; + new.color := new.color || ' trig modified'; + raise warning 'before insert (new, modified): %', new.*::text; + end if; + end if; + return new; +end; +$$; +create trigger upsert_before_trig before insert or update on upsert + for each row execute procedure upsert_before_func(); + +create function upsert_after_func() + returns trigger language plpgsql as +$$ +begin + if (TG_OP = 'UPDATE') then + raise warning 'after update (old): %', new.*::text; + raise warning 'after update (new): %', new.*::text; + elsif (TG_OP = 'INSERT') then + raise warning 'after insert (new): %', new.*::text; + end if; + return null; +end; +$$; +create trigger upsert_after_trig after insert or update on upsert + for each row execute procedure upsert_after_func(); + +insert into upsert values(1, 'black') on conflict (key) do update set color = 'updated ' || upsert.color; +insert into upsert values(2, 'red') on conflict (key) do update set color = 'updated ' || upsert.color; +insert into upsert values(3, 'orange') on conflict (key) do update set color = 'updated ' || upsert.color; +insert into upsert values(4, 'green') on conflict (key) do update set color = 'updated ' || upsert.color; +insert into upsert values(5, 'purple') on conflict (key) do update set color = 'updated ' || upsert.color; +insert into upsert values(6, 'white') on conflict (key) do update set color = 'updated ' || upsert.color; +insert into upsert values(7, 'pink') on conflict (key) do update set color = 'updated ' || upsert.color; +insert into upsert values(8, 'yellow') on conflict (key) do update set color = 'updated ' || upsert.color; + +select * from upsert; + +drop table upsert; +drop function upsert_before_func(); +drop function upsert_after_func(); diff --git a/src/test/regress/sql/updatable_views.sql b/src/test/regress/sql/updatable_views.sql index 697363665c1..8fe96f5c51c 100644 --- a/src/test/regress/sql/updatable_views.sql +++ b/src/test/regress/sql/updatable_views.sql @@ -69,6 +69,15 @@ DELETE FROM rw_view14 WHERE a=3; -- should be OK -- Partially updatable view INSERT INTO rw_view15 VALUES (3, 'ROW 3'); -- should fail INSERT INTO rw_view15 (a) VALUES (3); -- should be OK +INSERT INTO rw_view15 (a) VALUES (3) ON CONFLICT DO NOTHING; -- succeeds +SELECT * FROM rw_view15; +INSERT INTO rw_view15 (a) VALUES (3) ON CONFLICT (a) DO NOTHING; -- succeeds +SELECT * FROM rw_view15; +INSERT INTO rw_view15 (a) VALUES (3) ON CONFLICT (a) DO UPDATE set a = excluded.a; -- succeeds +SELECT * FROM rw_view15; +INSERT INTO rw_view15 (a) VALUES (3) ON CONFLICT (a) DO UPDATE set upper = 'blarg'; -- fails +SELECT * FROM rw_view15; +SELECT * FROM rw_view15; ALTER VIEW rw_view15 ALTER COLUMN upper SET DEFAULT 'NOT SET'; INSERT INTO rw_view15 (a) VALUES (4); -- should fail UPDATE rw_view15 SET upper='ROW 3' WHERE a=3; -- should fail diff --git a/src/test/regress/sql/update.sql b/src/test/regress/sql/update.sql index e71128c04dd..5637c68acf7 100644 --- a/src/test/regress/sql/update.sql +++ b/src/test/regress/sql/update.sql @@ -8,6 +8,11 @@ CREATE TABLE update_test ( c TEXT ); +CREATE TABLE upsert_test ( + a INT PRIMARY KEY, + b TEXT +); + INSERT INTO update_test VALUES (5, 10, 'foo'); INSERT INTO update_test(b, a) VALUES (15, 10); @@ -74,4 +79,20 @@ UPDATE update_test AS t SET b = update_test.b + 10 WHERE t.a = 10; UPDATE update_test SET c = repeat('x', 10000) WHERE c = 'car'; SELECT a, b, char_length(c) FROM update_test; +-- Test ON CONFLICT DO UPDATE +INSERT INTO upsert_test VALUES(1, 'Boo'); +-- uncorrelated sub-select: +WITH aaa AS (SELECT 1 AS a, 'Foo' AS b) INSERT INTO upsert_test + VALUES (1, 'Bar') ON CONFLICT(a) + DO UPDATE SET (b, a) = (SELECT b, a FROM aaa) RETURNING *; +-- correlated sub-select: +INSERT INTO upsert_test VALUES (1, 'Baz') ON CONFLICT(a) + DO UPDATE SET (b, a) = (SELECT b || ', Correlated', a from upsert_test i WHERE i.a = upsert_test.a) + RETURNING *; +-- correlated sub-select (EXCLUDED.* alias): +INSERT INTO upsert_test VALUES (1, 'Bat') ON CONFLICT(a) + DO UPDATE SET (b, a) = (SELECT b || ', Excluded', a from upsert_test i WHERE i.a = excluded.a) + RETURNING *; + DROP TABLE update_test; +DROP TABLE upsert_test; diff --git a/src/test/regress/sql/with.sql b/src/test/regress/sql/with.sql index 1687c119830..3fd55f96b3f 100644 --- a/src/test/regress/sql/with.sql +++ b/src/test/regress/sql/with.sql @@ -795,6 +795,63 @@ SELECT * FROM t LIMIT 10; SELECT * FROM y; +-- data-modifying WITH containing INSERT...ON CONFLICT DO UPDATE +CREATE TABLE z AS SELECT i AS k, (i || ' v')::text v FROM generate_series(1, 16, 3) i; +ALTER TABLE z ADD UNIQUE (k); + +WITH t AS ( + INSERT INTO z SELECT i, 'insert' + FROM generate_series(0, 16) i + ON CONFLICT (k) DO UPDATE SET v = z.v || ', now update' + RETURNING * +) +SELECT * FROM t JOIN y ON t.k = y.a ORDER BY a, k; + +-- Test EXCLUDED.* reference within CTE +WITH aa AS ( + INSERT INTO z VALUES(1, 5) ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v + WHERE z.k != EXCLUDED.k + RETURNING * +) +SELECT * FROM aa; + +-- New query/snapshot demonstrates side-effects of previous query. +SELECT * FROM z ORDER BY k; + +-- +-- Ensure subqueries within the update clause work, even if they +-- reference outside values +-- +WITH aa AS (SELECT 1 a, 2 b) +INSERT INTO z VALUES(1, 'insert') +ON CONFLICT (k) DO UPDATE SET v = (SELECT b || ' update' FROM aa WHERE a = 1 LIMIT 1); +WITH aa AS (SELECT 1 a, 2 b) +INSERT INTO z VALUES(1, 'insert') +ON CONFLICT (k) DO UPDATE SET v = ' update' WHERE z.k = (SELECT a FROM aa); +WITH aa AS (SELECT 1 a, 2 b) +INSERT INTO z VALUES(1, 'insert') +ON CONFLICT (k) DO UPDATE SET v = (SELECT b || ' update' FROM aa WHERE a = 1 LIMIT 1); +WITH aa AS (SELECT 'a' a, 'b' b UNION ALL SELECT 'a' a, 'b' b) +INSERT INTO z VALUES(1, 'insert') +ON CONFLICT (k) DO UPDATE SET v = (SELECT b || ' update' FROM aa WHERE a = 'a' LIMIT 1); +WITH aa AS (SELECT 1 a, 2 b) +INSERT INTO z VALUES(1, (SELECT b || ' insert' FROM aa WHERE a = 1 )) +ON CONFLICT (k) DO UPDATE SET v = (SELECT b || ' update' FROM aa WHERE a = 1 LIMIT 1); + +-- This shows an attempt to update an invisible row, which should really be +-- reported as a cardinality violation, but it doesn't seem worth fixing: +WITH simpletup AS ( + SELECT 2 k, 'Green' v), +upsert_cte AS ( + INSERT INTO z VALUES(2, 'Blue') ON CONFLICT (k) DO + UPDATE SET (k, v) = (SELECT k, v FROM simpletup WHERE simpletup.k = z.k) + RETURNING k, v) +INSERT INTO z VALUES(2, 'Red') ON CONFLICT (k) DO +UPDATE SET (k, v) = (SELECT k, v FROM upsert_cte WHERE upsert_cte.k = z.k) +RETURNING k, v; + +DROP TABLE z; + -- check that run to completion happens in proper ordering TRUNCATE TABLE y; |