aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/bin/pg_rewind/parsexlog.c33
-rw-r--r--src/bin/pg_rewind/pg_rewind.c87
-rw-r--r--src/bin/pg_rewind/pg_rewind.h6
-rw-r--r--src/bin/pg_rewind/t/001_basic.pl3
-rw-r--r--src/bin/pg_rewind/t/RewindTest.pm70
-rw-r--r--src/common/Makefile1
-rw-r--r--src/common/exec.c3
-rw-r--r--src/common/fe_archive.c128
-rw-r--r--src/include/common/fe_archive.h21
-rw-r--r--src/include/port.h3
-rw-r--r--src/tools/msvc/Mkvcbuild.pm4
11 files changed, 338 insertions, 21 deletions
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index eb61cb8803f..14a5db54331 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -19,6 +19,7 @@
#include "catalog/pg_control.h"
#include "catalog/storage_xlog.h"
#include "commands/dbcommands_xlog.h"
+#include "common/fe_archive.h"
#include "filemap.h"
#include "pg_rewind.h"
@@ -41,6 +42,7 @@ static char xlogfpath[MAXPGPATH];
typedef struct XLogPageReadPrivate
{
+ const char *restoreCommand;
int tliIndex;
} XLogPageReadPrivate;
@@ -55,7 +57,7 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader,
*/
void
extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
- XLogRecPtr endpoint)
+ XLogRecPtr endpoint, const char *restoreCommand)
{
XLogRecord *record;
XLogReaderState *xlogreader;
@@ -63,6 +65,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogPageReadPrivate private;
private.tliIndex = tliIndex;
+ private.restoreCommand = restoreCommand;
xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
&private);
if (xlogreader == NULL)
@@ -146,7 +149,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
void
findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
- XLogRecPtr *lastchkptredo)
+ XLogRecPtr *lastchkptredo, const char *restoreCommand)
{
/* Walk backwards, starting from the given record */
XLogRecord *record;
@@ -170,6 +173,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
private.tliIndex = tliIndex;
+ private.restoreCommand = restoreCommand;
xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
&private);
if (xlogreader == NULL)
@@ -281,8 +285,29 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
- pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ /*
+ * If we have no restore_command to execute, then exit.
+ */
+ if (private->restoreCommand == NULL)
+ {
+ pg_log_error("could not open file \"%s\": %m", xlogfpath);
+ return -1;
+ }
+
+ /*
+ * Since we have restore_command, then try to retrieve missing WAL
+ * file from the archive.
+ */
+ xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
+ xlogfname,
+ WalSegSz,
+ private->restoreCommand);
+
+ if (xlogreadfd < 0)
+ return -1;
+ else
+ pg_log_debug("using file \"%s\" restored from archive",
+ xlogfpath);
}
}
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c
index bf2d2983e7e..101f0911bec 100644
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@ -22,6 +22,7 @@
#include "common/file_perm.h"
#include "common/file_utils.h"
#include "common/restricted_token.h"
+#include "common/string.h"
#include "fe_utils/recovery_gen.h"
#include "fetch.h"
#include "file_ops.h"
@@ -38,6 +39,7 @@ static void createBackupLabel(XLogRecPtr startpoint, TimeLineID starttli,
static void digestControlFile(ControlFileData *ControlFile, char *source,
size_t size);
static void syncTargetDirectory(void);
+static void getRestoreCommand(const char *argv0);
static void sanityChecks(void);
static void findCommonAncestorTimeline(XLogRecPtr *recptr, int *tliIndex);
static void ensureCleanShutdown(const char *argv0);
@@ -53,11 +55,13 @@ int WalSegSz;
char *datadir_target = NULL;
char *datadir_source = NULL;
char *connstr_source = NULL;
+char *restore_command = NULL;
static bool debug = false;
bool showprogress = false;
bool dry_run = false;
bool do_sync = true;
+bool restore_wal = false;
/* Target history */
TimeLineHistoryEntry *targetHistory;
@@ -74,6 +78,8 @@ usage(const char *progname)
printf(_("%s resynchronizes a PostgreSQL cluster with another copy of the cluster.\n\n"), progname);
printf(_("Usage:\n %s [OPTION]...\n\n"), progname);
printf(_("Options:\n"));
+ printf(_(" -c, --restore-target-wal use restore_command in target config\n"
+ " to retrieve WAL files from archives\n"));
printf(_(" -D, --target-pgdata=DIRECTORY existing data directory to modify\n"));
printf(_(" --source-pgdata=DIRECTORY source data directory to synchronize with\n"));
printf(_(" --source-server=CONNSTR source server to synchronize with\n"));
@@ -103,6 +109,7 @@ main(int argc, char **argv)
{"source-server", required_argument, NULL, 2},
{"no-ensure-shutdown", no_argument, NULL, 4},
{"version", no_argument, NULL, 'V'},
+ {"restore-target-wal", no_argument, NULL, 'c'},
{"dry-run", no_argument, NULL, 'n'},
{"no-sync", no_argument, NULL, 'N'},
{"progress", no_argument, NULL, 'P'},
@@ -144,7 +151,7 @@ main(int argc, char **argv)
}
}
- while ((c = getopt_long(argc, argv, "D:nNPR", long_options, &option_index)) != -1)
+ while ((c = getopt_long(argc, argv, "cD:nNPR", long_options, &option_index)) != -1)
{
switch (c)
{
@@ -152,6 +159,10 @@ main(int argc, char **argv)
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
+ case 'c':
+ restore_wal = true;
+ break;
+
case 'P':
showprogress = true;
break;
@@ -255,6 +266,8 @@ main(int argc, char **argv)
umask(pg_mode_mask);
+ getRestoreCommand(argv[0]);
+
atexit(disconnect_atexit);
/* Connect to remote server */
@@ -350,9 +363,8 @@ main(int argc, char **argv)
exit(0);
}
- findLastCheckpoint(datadir_target, divergerec,
- lastcommontliIndex,
- &chkptrec, &chkpttli, &chkptredo);
+ findLastCheckpoint(datadir_target, divergerec, lastcommontliIndex,
+ &chkptrec, &chkpttli, &chkptredo, restore_command);
pg_log_info("rewinding from last common checkpoint at %X/%X on timeline %u",
(uint32) (chkptrec >> 32), (uint32) chkptrec,
chkpttli);
@@ -378,7 +390,7 @@ main(int argc, char **argv)
if (showprogress)
pg_log_info("reading WAL in target");
extractPageMap(datadir_target, chkptrec, lastcommontliIndex,
- ControlFile_target.checkPoint);
+ ControlFile_target.checkPoint, restore_command);
filemap_finalize();
if (showprogress)
@@ -805,6 +817,71 @@ syncTargetDirectory(void)
}
/*
+ * Get value of GUC parameter restore_command from the target cluster.
+ *
+ * This uses a logic based on "postgres -C" to get the value from the
+ * cluster.
+ */
+static void
+getRestoreCommand(const char *argv0)
+{
+ int rc;
+ char postgres_exec_path[MAXPGPATH],
+ postgres_cmd[MAXPGPATH],
+ cmd_output[MAXPGPATH];
+
+ if (!restore_wal)
+ return;
+
+ /* find postgres executable */
+ rc = find_other_exec(argv0, "postgres",
+ PG_BACKEND_VERSIONSTR,
+ postgres_exec_path);
+
+ if (rc < 0)
+ {
+ char full_path[MAXPGPATH];
+
+ if (find_my_exec(argv0, full_path) < 0)
+ strlcpy(full_path, progname, sizeof(full_path));
+
+ if (rc == -1)
+ pg_log_error("The program \"postgres\" is needed by %s but was not found in the\n"
+ "same directory as \"%s\".\n"
+ "Check your installation.",
+ progname, full_path);
+ else
+ pg_log_error("The program \"postgres\" was found by \"%s\"\n"
+ "but was not the same version as %s.\n"
+ "Check your installation.",
+ full_path, progname);
+ exit(1);
+ }
+
+ /*
+ * Build a command able to retrieve the value of GUC parameter
+ * restore_command, if set.
+ */
+ snprintf(postgres_cmd, sizeof(postgres_cmd),
+ "\"%s\" -D \"%s\" -C restore_command",
+ postgres_exec_path, datadir_target);
+
+ if (!pipe_read_line(postgres_cmd, cmd_output, sizeof(cmd_output)))
+ exit(1);
+
+ (void) pg_strip_crlf(cmd_output);
+
+ if (strcmp(cmd_output, "") == 0)
+ pg_fatal("restore_command is not set on the target cluster");
+
+ restore_command = pg_strdup(cmd_output);
+
+ pg_log_debug("using for rewind restore_command = \'%s\'",
+ restore_command);
+}
+
+
+/*
* Ensure clean shutdown of target instance by launching single-user mode
* postgres to do crash recovery.
*/
diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h
index e4e8d23c32d..b122ae43e5a 100644
--- a/src/bin/pg_rewind/pg_rewind.h
+++ b/src/bin/pg_rewind/pg_rewind.h
@@ -42,11 +42,13 @@ extern uint64 fetch_done;
/* in parsexlog.c */
extern void extractPageMap(const char *datadir, XLogRecPtr startpoint,
- int tliIndex, XLogRecPtr endpoint);
+ int tliIndex, XLogRecPtr endpoint,
+ const char *restoreCommand);
extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr,
int tliIndex,
XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
- XLogRecPtr *lastchkptredo);
+ XLogRecPtr *lastchkptredo,
+ const char *restoreCommand);
extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr,
int tliIndex);
diff --git a/src/bin/pg_rewind/t/001_basic.pl b/src/bin/pg_rewind/t/001_basic.pl
index 95d8ccfced1..d97e4377419 100644
--- a/src/bin/pg_rewind/t/001_basic.pl
+++ b/src/bin/pg_rewind/t/001_basic.pl
@@ -1,7 +1,7 @@
use strict;
use warnings;
use TestLib;
-use Test::More tests => 15;
+use Test::More tests => 20;
use FindBin;
use lib $FindBin::RealBin;
@@ -171,5 +171,6 @@ in master, before promotion
# Run the test in both modes
run_test('local');
run_test('remote');
+run_test('archive');
exit(0);
diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm
index 82fa220ac86..7dabf395e10 100644
--- a/src/bin/pg_rewind/t/RewindTest.pm
+++ b/src/bin/pg_rewind/t/RewindTest.pm
@@ -38,6 +38,7 @@ use File::Copy;
use File::Path qw(rmtree);
use IPC::Run qw(run);
use PostgresNode;
+use RecursiveCopy;
use TestLib;
use Test::More;
@@ -227,10 +228,26 @@ sub run_pg_rewind
# Append the rewind-specific role to the connection string.
$standby_connstr = "$standby_connstr user=rewind_user";
- # Stop the master and be ready to perform the rewind. The cluster
- # needs recovery to finish once, and pg_rewind makes sure that it
- # happens automatically.
- $node_master->stop('immediate');
+ if ($test_mode eq 'archive')
+ {
+ # pg_rewind is tested with --restore-target-wal by moving all
+ # WAL files to a secondary location. Note that this leads to
+ # a failure in ensureCleanShutdown(), forcing to the use of
+ # --no-ensure-shutdown in this mode as the initial set of WAL
+ # files needed to ensure a clean restart is gone. This could
+ # be improved by keeping around only a minimum set of WAL
+ # segments but that would just make the test more costly,
+ # without improving the coverage. Hence, instead, stop
+ # gracefully the primary here.
+ $node_master->stop;
+ }
+ else
+ {
+ # Stop the master and be ready to perform the rewind. The cluster
+ # needs recovery to finish once, and pg_rewind makes sure that it
+ # happens automatically.
+ $node_master->stop('immediate');
+ }
# At this point, the rewind processing is ready to run.
# We now have a very simple scenario with a few diverged WAL record.
@@ -284,6 +301,51 @@ sub run_pg_rewind
$node_standby->safe_psql('postgres',
"ALTER ROLE rewind_user WITH REPLICATION;");
}
+ elsif ($test_mode eq "archive")
+ {
+
+ # Do rewind using a local pgdata as source and specified
+ # directory with target WAL archive. The old master has
+ # to be stopped at this point.
+
+ # Remove the existing archive directory and move all WAL
+ # segments from the old master to the archives. These
+ # will be used by pg_rewind.
+ rmtree($node_master->archive_dir);
+ RecursiveCopy::copypath($node_master->data_dir . "/pg_wal",
+ $node_master->archive_dir);
+
+ # Fast way to remove entire directory content
+ rmtree($node_master->data_dir . "/pg_wal");
+ mkdir($node_master->data_dir . "/pg_wal");
+
+ # Make sure that directories have the right umask as this is
+ # required by a follow-up check on permissions, and better
+ # safe than sorry.
+ chmod(0700, $node_master->archive_dir);
+ chmod(0700, $node_master->data_dir . "/pg_wal");
+
+ # Add appropriate restore_command to the target cluster
+ $node_master->enable_restoring($node_master, 0);
+
+ # Stop the new master and be ready to perform the rewind.
+ $node_standby->stop;
+
+ # Note the use of --no-ensure-shutdown here. WAL files are
+ # gone in this mode and the primary has been stopped
+ # gracefully already.
+ command_ok(
+ [
+ 'pg_rewind',
+ "--debug",
+ "--source-pgdata=$standby_pgdata",
+ "--target-pgdata=$master_pgdata",
+ "--no-sync",
+ "--no-ensure-shutdown",
+ "--restore-target-wal"
+ ],
+ 'pg_rewind archive');
+ }
else
{
diff --git a/src/common/Makefile b/src/common/Makefile
index 6939b9d0874..a97c723fbda 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -88,6 +88,7 @@ endif
# (Mkvcbuild.pm has a copy of this list, too)
OBJS_FRONTEND = \
$(OBJS_COMMON) \
+ fe_archive.o \
fe_memutils.o \
file_utils.o \
logging.o \
diff --git a/src/common/exec.c b/src/common/exec.c
index 88400daa471..f39b0a294bf 100644
--- a/src/common/exec.c
+++ b/src/common/exec.c
@@ -51,7 +51,6 @@
static int validate_exec(const char *path);
static int resolve_symlinks(char *path);
-static char *pipe_read_line(char *cmd, char *line, int maxsize);
#ifdef WIN32
static BOOL GetTokenUser(HANDLE hToken, PTOKEN_USER *ppTokenUser);
@@ -356,7 +355,7 @@ find_other_exec(const char *argv0, const char *target,
/*
* Execute a command in a pipe and read the first line from it.
*/
-static char *
+char *
pipe_read_line(char *cmd, char *line, int maxsize)
{
FILE *pgver;
diff --git a/src/common/fe_archive.c b/src/common/fe_archive.c
new file mode 100644
index 00000000000..b0d68870db8
--- /dev/null
+++ b/src/common/fe_archive.c
@@ -0,0 +1,128 @@
+/*-------------------------------------------------------------------------
+ *
+ * fe_archive.c
+ * Routines to access WAL archives from frontend
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/common/fe_archive.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#error "This file is not expected to be compiled for backend code"
+#endif
+
+#include "postgres_fe.h"
+
+#include <unistd.h>
+#include <sys/stat.h>
+
+#include "access/xlog_internal.h"
+#include "common/archive.h"
+#include "common/fe_archive.h"
+#include "common/logging.h"
+
+
+/*
+ * RestoreArchivedFile
+ *
+ * Attempt to retrieve the specified file from off-line archival storage.
+ * If successful, return a file descriptor of the restored file, else
+ * return -1.
+ *
+ * For fixed-size files, the caller may pass the expected size as an
+ * additional crosscheck on successful recovery. If the file size is not
+ * known, set expectedSize = 0.
+ */
+int
+RestoreArchivedFile(const char *path, const char *xlogfname,
+ off_t expectedSize, const char *restoreCommand)
+{
+ char xlogpath[MAXPGPATH];
+ char *xlogRestoreCmd;
+ int rc;
+ struct stat stat_buf;
+
+ snprintf(xlogpath, MAXPGPATH, "%s/" XLOGDIR "/%s", path, xlogfname);
+
+ xlogRestoreCmd = BuildRestoreCommand(restoreCommand, xlogpath,
+ xlogfname, NULL);
+ if (xlogRestoreCmd == NULL)
+ {
+ pg_log_fatal("could not use restore_command with %%r alias");
+ exit(1);
+ }
+
+ /*
+ * Execute restore_command, which should copy the missing file from
+ * archival storage.
+ */
+ rc = system(xlogRestoreCmd);
+ pfree(xlogRestoreCmd);
+
+ if (rc == 0)
+ {
+ /*
+ * Command apparently succeeded, but let's make sure the file is
+ * really there now and has the correct size.
+ */
+ if (stat(xlogpath, &stat_buf) == 0)
+ {
+ if (expectedSize > 0 && stat_buf.st_size != expectedSize)
+ {
+ pg_log_fatal("unexpected file size for \"%s\": %lu instead of %lu",
+ xlogfname, (unsigned long) stat_buf.st_size,
+ (unsigned long) expectedSize);
+ exit(1);
+ }
+ else
+ {
+ int xlogfd = open(xlogpath, O_RDONLY | PG_BINARY, 0);
+
+ if (xlogfd < 0)
+ {
+ pg_log_fatal("could not open file \"%s\" restored from archive: %m",
+ xlogpath);
+ exit(1);
+ }
+ else
+ return xlogfd;
+ }
+ }
+ else
+ {
+ if (errno != ENOENT)
+ {
+ pg_log_fatal("could not stat file \"%s\": %m",
+ xlogpath);
+ exit(1);
+ }
+ }
+ }
+
+ /*
+ * If the failure was due to a signal, then it would be misleading to
+ * return with a failure at restoring the file. So just bail out and
+ * exit. Hard shell errors such as "command not found" are treated as
+ * fatal too.
+ */
+ if (wait_result_is_any_signal(rc, true))
+ {
+ pg_log_fatal("restore_command failed due to the signal: %s",
+ wait_result_to_str(rc));
+ exit(1);
+ }
+
+ /*
+ * The file is not available, so just let the caller decide what to do
+ * next.
+ */
+ pg_log_error("could not restore file \"%s\" from archive",
+ xlogfname);
+ return -1;
+}
diff --git a/src/include/common/fe_archive.h b/src/include/common/fe_archive.h
new file mode 100644
index 00000000000..495b560d245
--- /dev/null
+++ b/src/include/common/fe_archive.h
@@ -0,0 +1,21 @@
+/*-------------------------------------------------------------------------
+ *
+ * fe_archive.h
+ * Routines to access WAL archives from frontend
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/fe_archive.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FE_ARCHIVE_H
+#define FE_ARCHIVE_H
+
+extern int RestoreArchivedFile(const char *path,
+ const char *xlogfname,
+ off_t expectedSize,
+ const char *restoreCommand);
+
+#endif /* FE_ARCHIVE_H */
diff --git a/src/include/port.h b/src/include/port.h
index 29f3e39e5b3..20a5de1b7a7 100644
--- a/src/include/port.h
+++ b/src/include/port.h
@@ -102,10 +102,11 @@ extern void pgfnames_cleanup(char **filenames);
/* Portable locale initialization (in exec.c) */
extern void set_pglocale_pgservice(const char *argv0, const char *app);
-/* Portable way to find binaries (in exec.c) */
+/* Portable way to find and execute binaries (in exec.c) */
extern int find_my_exec(const char *argv0, char *retpath);
extern int find_other_exec(const char *argv0, const char *target,
const char *versionstr, char *retpath);
+extern char *pipe_read_line(char *cmd, char *line, int maxsize);
/* Doesn't belong here, but this is used with find_other_exec(), so... */
#define PG_BACKEND_VERSIONSTR "postgres (PostgreSQL) " PG_VERSION "\n"
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 39709f20e67..5c88825f496 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -139,8 +139,8 @@ sub mkvcbuild
}
our @pgcommonfrontendfiles = (
- @pgcommonallfiles, qw(fe_memutils.c file_utils.c
- logging.c restricted_token.c));
+ @pgcommonallfiles, qw(fe_archive.c fe_memutils.c
+ file_utils.c logging.c restricted_token.c));
our @pgcommonbkndfiles = @pgcommonallfiles;
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-25 06:37:11 +0000