To support external compression of archived WAL data, add a flag bit to

WAL records that shows whether it is safe to remove full-page images
(ie, whether or not an on-line backup was in progress when the WAL entry
was made).  Also make provision for an XLOG_NOOP record type that can be
used to fill in the extra space when decompressing the data for restore.

This is the portion of Koichi Suzuki's "full page writes" patch that
has to go into the core database.  The remainder of that work is two
external compression and decompression programs, which for the time being
will undergo separate development on pgfoundry.  Per discussion.

Also, twiddle the handling of BTREE_SPLIT records to ensure it'll be
possible to compress them (the previous coding caused essential info
to be omitted).  The other commonly-used record types seem OK already,
with the possible exception of GIN and GIST WAL records, which I don't
understand well enough to opine on.

3 files changed, 38 insertions, 11 deletions

diff --git a/src/backend/access/nbtree/nbtinsert.c b/src/backend/access/nbtree/nbtinsert.c
index 775eaca2427..5f8be398ca1 100644
--- a/src/backend/access/nbtree/nbtinsert.c
+++ b/src/backend/access/nbtree/nbtinsert.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtinsert.c,v 1.156 2007/04/11 20:47:37 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtinsert.c,v 1.157 2007/05/20 21:08:19 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1034,21 +1034,23 @@ _bt_split(Relation rel, Buffer buf, OffsetNumber firstright,
 		 * Log the new item and its offset, if it was inserted on the left
 		 * page. (If it was put on the right page, we don't need to explicitly
 		 * WAL log it because it's included with all the other items on the
-		 * right page.) Show these as belonging to the left page buffer,
-		 * so that they are not stored if XLogInsert decides it needs a
-		 * full-page image of the left page.
+		 * right page.) Show the new item as belonging to the left page buffer,
+		 * so that it is not stored if XLogInsert decides it needs a full-page
+		 * image of the left page.  We store the offset anyway, though, to
+		 * support archive compression of these records.
 		 */
 		if (newitemonleft)
 		{
 			lastrdata->next = lastrdata + 1;
 			lastrdata++;
+
 			lastrdata->data = (char *) &newitemoff;
 			lastrdata->len = sizeof(OffsetNumber);
-			lastrdata->buffer = buf;		/* backup block 1 */
-			lastrdata->buffer_std = true;
+			lastrdata->buffer = InvalidBuffer;
 
 			lastrdata->next = lastrdata + 1;
 			lastrdata++;
+
 			lastrdata->data = (char *) newitem;
 			lastrdata->len = MAXALIGN(newitemsz);
 			lastrdata->buffer = buf;		/* backup block 1 */
@@ -1064,6 +1066,7 @@ _bt_split(Relation rel, Buffer buf, OffsetNumber firstright,
 			 */
 			lastrdata->next = lastrdata + 1;
 			lastrdata++;
+
 			lastrdata->data = NULL;
 			lastrdata->len = 0;
 			lastrdata->buffer = buf;		/* backup block 1 */
diff --git a/src/backend/access/nbtree/nbtxlog.c b/src/backend/access/nbtree/nbtxlog.c
index ff41be37679..189695853a5 100644
--- a/src/backend/access/nbtree/nbtxlog.c
+++ b/src/backend/access/nbtree/nbtxlog.c
@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtxlog.c,v 1.43 2007/04/11 20:47:38 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtxlog.c,v 1.44 2007/05/20 21:08:19 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -292,14 +292,17 @@ btree_xlog_split(bool onleft, bool isroot,
 	}
 
 	/* Extract newitem and newitemoff, if present */
-	if (onleft && !(record->xl_info & XLR_BKP_BLOCK_1))
+	if (onleft)
 	{
-		IndexTupleData itupdata;
-
 		/* Extract the offset (still assuming 16-bit alignment) */
 		memcpy(&newitemoff, datapos, sizeof(OffsetNumber));
 		datapos += sizeof(OffsetNumber);
 		datalen -= sizeof(OffsetNumber);
+	}
+
+	if (onleft && !(record->xl_info & XLR_BKP_BLOCK_1))
+	{
+		IndexTupleData itupdata;
 
 		/*
 		 * We need to copy the tuple header to apply IndexTupleDSize, because
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 655563a3d84..3dc00499bfb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.268 2007/04/30 21:01:52 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.269 2007/05/20 21:08:19 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -784,6 +784,19 @@ begin:;
 	}
 
 	/*
+	 * If we backed up any full blocks and online backup is not in progress,
+	 * mark the backup blocks as removable.  This allows the WAL archiver to
+	 * know whether it is safe to compress archived WAL data by transforming
+	 * full-block records into the non-full-block format.
+	 *
+	 * Note: we could just set the flag whenever !forcePageWrites, but
+	 * defining it like this leaves the info bit free for some potential
+	 * other use in records without any backup blocks.
+	 */
+	if ((info & XLR_BKP_BLOCK_MASK) && !Insert->forcePageWrites)
+		info |= XLR_BKP_REMOVABLE;
+
+	/*
 	 * If there isn't enough space on the current XLOG page for a record
 	 * header, advance to the next page (leaving the unused space as zeroes).
 	 */
@@ -5868,6 +5881,10 @@ xlog_redo(XLogRecPtr lsn, XLogRecord *record)
 
 		RecoveryRestartPoint(&checkPoint);
 	}
+	else if (info == XLOG_NOOP)
+	{
+		/* nothing to do here */
+	}
 	else if (info == XLOG_SWITCH)
 	{
 		/* nothing to do here */
@@ -5894,6 +5911,10 @@ xlog_desc(StringInfo buf, uint8 xl_info, char *rec)
 						 checkpoint->nextMultiOffset,
 				 (info == XLOG_CHECKPOINT_SHUTDOWN) ? "shutdown" : "online");
 	}
+	else if (info == XLOG_NOOP)
+	{
+		appendStringInfo(buf, "xlog no-op");
+	}
 	else if (info == XLOG_NEXTOID)
 	{
 		Oid			nextOid;