From 6dd263cfaa8447470af4fae3f61c47438f91d71f Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 13 Mar 2019 10:43:20 +0900 Subject: Rename pg_verify_checksums to pg_checksums The current tool name is too restrictive and focuses only on verifying checksums. As more options to control checksums for an offline cluster are planned to be added, switch to a more generic name. Documentation as well as all past references to the tool are updated. Author: Michael Paquier Reviewed-by: Michael Banck, Fabien Coelho, Seigei Kornilov Discussion: https://postgr.es/m/20181221201616.GD4974@nighthawk.caipicrew.dd-dns.de --- src/backend/replication/basebackup.c | 2 +- src/bin/Makefile | 2 +- src/bin/initdb/t/001_initdb.pl | 8 +- src/bin/pg_checksums/.gitignore | 3 + src/bin/pg_checksums/Makefile | 42 +++ src/bin/pg_checksums/nls.mk | 4 + src/bin/pg_checksums/pg_checksums.c | 358 ++++++++++++++++++++++ src/bin/pg_checksums/t/001_basic.pl | 8 + src/bin/pg_checksums/t/002_actions.pl | 160 ++++++++++ src/bin/pg_verify_checksums/.gitignore | 3 - src/bin/pg_verify_checksums/Makefile | 42 --- src/bin/pg_verify_checksums/nls.mk | 4 - src/bin/pg_verify_checksums/pg_verify_checksums.c | 354 --------------------- src/bin/pg_verify_checksums/t/001_basic.pl | 8 - src/bin/pg_verify_checksums/t/002_actions.pl | 160 ---------- 15 files changed, 581 insertions(+), 577 deletions(-) create mode 100644 src/bin/pg_checksums/.gitignore create mode 100644 src/bin/pg_checksums/Makefile create mode 100644 src/bin/pg_checksums/nls.mk create mode 100644 src/bin/pg_checksums/pg_checksums.c create mode 100644 src/bin/pg_checksums/t/001_basic.pl create mode 100644 src/bin/pg_checksums/t/002_actions.pl delete mode 100644 src/bin/pg_verify_checksums/.gitignore delete mode 100644 src/bin/pg_verify_checksums/Makefile delete mode 100644 src/bin/pg_verify_checksums/nls.mk delete mode 100644 src/bin/pg_verify_checksums/pg_verify_checksums.c delete mode 100644 src/bin/pg_verify_checksums/t/001_basic.pl delete mode 100644 src/bin/pg_verify_checksums/t/002_actions.pl (limited to 'src') diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 6c324a6661d..537f09e3429 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -190,7 +190,7 @@ static const char *excludeFiles[] = /* * List of files excluded from checksum validation. * - * Note: this list should be kept in sync with what pg_verify_checksums.c + * Note: this list should be kept in sync with what pg_checksums.c * includes. */ static const char *const noChecksumFiles[] = { diff --git a/src/bin/Makefile b/src/bin/Makefile index c66bfa887ee..903e58121f6 100644 --- a/src/bin/Makefile +++ b/src/bin/Makefile @@ -17,6 +17,7 @@ SUBDIRS = \ initdb \ pg_archivecleanup \ pg_basebackup \ + pg_checksums \ pg_config \ pg_controldata \ pg_ctl \ @@ -26,7 +27,6 @@ SUBDIRS = \ pg_test_fsync \ pg_test_timing \ pg_upgrade \ - pg_verify_checksums \ pg_waldump \ pgbench \ psql \ diff --git a/src/bin/initdb/t/001_initdb.pl b/src/bin/initdb/t/001_initdb.pl index 759779adb24..8dfcd8752a1 100644 --- a/src/bin/initdb/t/001_initdb.pl +++ b/src/bin/initdb/t/001_initdb.pl @@ -63,12 +63,12 @@ mkdir $datadir; command_like(['pg_controldata', $datadir], qr/Data page checksum version:.*0/, 'checksums are disabled in control file'); -# pg_verify_checksums fails with checksums disabled by default. This is -# not part of the tests included in pg_verify_checksums to save from +# pg_checksums fails with checksums disabled by default. This is +# not part of the tests included in pg_checksums to save from # the creation of an extra instance. command_fails( - [ 'pg_verify_checksums', '-D', $datadir], - "pg_verify_checksums fails with data checksum disabled"); + [ 'pg_checksums', '-D', $datadir], + "pg_checksums fails with data checksum disabled"); command_ok([ 'initdb', '-S', $datadir ], 'sync only'); command_fails([ 'initdb', $datadir ], 'existing data directory'); diff --git a/src/bin/pg_checksums/.gitignore b/src/bin/pg_checksums/.gitignore new file mode 100644 index 00000000000..78886250942 --- /dev/null +++ b/src/bin/pg_checksums/.gitignore @@ -0,0 +1,3 @@ +/pg_checksums + +/tmp_check/ diff --git a/src/bin/pg_checksums/Makefile b/src/bin/pg_checksums/Makefile new file mode 100644 index 00000000000..278b7a0f2ec --- /dev/null +++ b/src/bin/pg_checksums/Makefile @@ -0,0 +1,42 @@ +#------------------------------------------------------------------------- +# +# Makefile for src/bin/pg_checksums +# +# Copyright (c) 1998-2019, PostgreSQL Global Development Group +# +# src/bin/pg_checksums/Makefile +# +#------------------------------------------------------------------------- + +PGFILEDESC = "pg_checksums - verify data checksums in an offline cluster" +PGAPPICON=win32 + +subdir = src/bin/pg_checksums +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +OBJS= pg_checksums.o $(WIN32RES) + +all: pg_checksums + +pg_checksums: $(OBJS) | submake-libpgport + $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) + +install: all installdirs + $(INSTALL_PROGRAM) pg_checksums$(X) '$(DESTDIR)$(bindir)/pg_checksums$(X)' + +installdirs: + $(MKDIR_P) '$(DESTDIR)$(bindir)' + +uninstall: + rm -f '$(DESTDIR)$(bindir)/pg_checksums$(X)' + +clean distclean maintainer-clean: + rm -f pg_checksums$(X) $(OBJS) + rm -rf tmp_check + +check: + $(prove_check) + +installcheck: + $(prove_installcheck) diff --git a/src/bin/pg_checksums/nls.mk b/src/bin/pg_checksums/nls.mk new file mode 100644 index 00000000000..2748b18ef72 --- /dev/null +++ b/src/bin/pg_checksums/nls.mk @@ -0,0 +1,4 @@ +# src/bin/pg_checksums/nls.mk +CATALOG_NAME = pg_checksums +AVAIL_LANGUAGES = +GETTEXT_FILES = pg_checksums.c diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c new file mode 100644 index 00000000000..6571c342111 --- /dev/null +++ b/src/bin/pg_checksums/pg_checksums.c @@ -0,0 +1,358 @@ +/*------------------------------------------------------------------------- + * + * pg_checksums.c + * Verifies page level checksums in an offline cluster. + * + * Copyright (c) 2010-2019, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_checksums/pg_checksums.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include +#include +#include + +#include "catalog/pg_control.h" +#include "common/controldata_utils.h" +#include "getopt_long.h" +#include "pg_getopt.h" +#include "storage/bufpage.h" +#include "storage/checksum.h" +#include "storage/checksum_impl.h" +#include "storage/fd.h" + + +static int64 files = 0; +static int64 blocks = 0; +static int64 badblocks = 0; +static ControlFileData *ControlFile; + +static char *only_relfilenode = NULL; +static bool verbose = false; + +static const char *progname; + +static void +usage(void) +{ + printf(_("%s verifies data checksums in a PostgreSQL database cluster.\n\n"), progname); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]... [DATADIR]\n"), progname); + printf(_("\nOptions:\n")); + printf(_(" [-D, --pgdata=]DATADIR data directory\n")); + printf(_(" -v, --verbose output verbose messages\n")); + printf(_(" -r RELFILENODE check only relation with specified relfilenode\n")); + printf(_(" -V, --version output version information, then exit\n")); + printf(_(" -?, --help show this help, then exit\n")); + printf(_("\nIf no data directory (DATADIR) is specified, " + "the environment variable PGDATA\nis used.\n\n")); + printf(_("Report bugs to .\n")); +} + +/* + * List of files excluded from checksum validation. + * + * Note: this list should be kept in sync with what basebackup.c includes. + */ +static const char *const skip[] = { + "pg_control", + "pg_filenode.map", + "pg_internal.init", + "PG_VERSION", +#ifdef EXEC_BACKEND + "config_exec_params", + "config_exec_params.new", +#endif + NULL, +}; + +static bool +skipfile(const char *fn) +{ + const char *const *f; + + for (f = skip; *f; f++) + if (strcmp(*f, fn) == 0) + return true; + + return false; +} + +static void +scan_file(const char *fn, BlockNumber segmentno) +{ + PGAlignedBlock buf; + PageHeader header = (PageHeader) buf.data; + int f; + BlockNumber blockno; + + f = open(fn, O_RDONLY | PG_BINARY, 0); + if (f < 0) + { + fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), + progname, fn, strerror(errno)); + exit(1); + } + + files++; + + for (blockno = 0;; blockno++) + { + uint16 csum; + int r = read(f, buf.data, BLCKSZ); + + if (r == 0) + break; + if (r != BLCKSZ) + { + fprintf(stderr, _("%s: could not read block %u in file \"%s\": read %d of %d\n"), + progname, blockno, fn, r, BLCKSZ); + exit(1); + } + blocks++; + + /* New pages have no checksum yet */ + if (PageIsNew(header)) + continue; + + csum = pg_checksum_page(buf.data, blockno + segmentno * RELSEG_SIZE); + if (csum != header->pd_checksum) + { + if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION) + fprintf(stderr, _("%s: checksum verification failed in file \"%s\", block %u: calculated checksum %X but block contains %X\n"), + progname, fn, blockno, csum, header->pd_checksum); + badblocks++; + } + } + + if (verbose) + fprintf(stderr, + _("%s: checksums verified in file \"%s\"\n"), progname, fn); + + close(f); +} + +static void +scan_directory(const char *basedir, const char *subdir) +{ + char path[MAXPGPATH]; + DIR *dir; + struct dirent *de; + + snprintf(path, sizeof(path), "%s/%s", basedir, subdir); + dir = opendir(path); + if (!dir) + { + fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), + progname, path, strerror(errno)); + exit(1); + } + while ((de = readdir(dir)) != NULL) + { + char fn[MAXPGPATH]; + struct stat st; + + if (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0) + continue; + + /* Skip temporary files */ + if (strncmp(de->d_name, + PG_TEMP_FILE_PREFIX, + strlen(PG_TEMP_FILE_PREFIX)) == 0) + continue; + + /* Skip temporary folders */ + if (strncmp(de->d_name, + PG_TEMP_FILES_DIR, + strlen(PG_TEMP_FILES_DIR)) == 0) + return; + + snprintf(fn, sizeof(fn), "%s/%s", path, de->d_name); + if (lstat(fn, &st) < 0) + { + fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), + progname, fn, strerror(errno)); + exit(1); + } + if (S_ISREG(st.st_mode)) + { + char fnonly[MAXPGPATH]; + char *forkpath, + *segmentpath; + BlockNumber segmentno = 0; + + if (skipfile(de->d_name)) + continue; + + /* + * Cut off at the segment boundary (".") to get the segment number + * in order to mix it into the checksum. Then also cut off at the + * fork boundary, to get the relfilenode the file belongs to for + * filtering. + */ + strlcpy(fnonly, de->d_name, sizeof(fnonly)); + segmentpath = strchr(fnonly, '.'); + if (segmentpath != NULL) + { + *segmentpath++ = '\0'; + segmentno = atoi(segmentpath); + if (segmentno == 0) + { + fprintf(stderr, _("%s: invalid segment number %d in file name \"%s\"\n"), + progname, segmentno, fn); + exit(1); + } + } + + forkpath = strchr(fnonly, '_'); + if (forkpath != NULL) + *forkpath++ = '\0'; + + if (only_relfilenode && strcmp(only_relfilenode, fnonly) != 0) + /* Relfilenode not to be included */ + continue; + + scan_file(fn, segmentno); + } +#ifndef WIN32 + else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)) +#else + else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn)) +#endif + scan_directory(path, de->d_name); + } + closedir(dir); +} + +int +main(int argc, char *argv[]) +{ + static struct option long_options[] = { + {"pgdata", required_argument, NULL, 'D'}, + {"verbose", no_argument, NULL, 'v'}, + {NULL, 0, NULL, 0} + }; + + char *DataDir = NULL; + int c; + int option_index; + bool crc_ok; + + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_checksums")); + + progname = get_progname(argv[0]); + + if (argc > 1) + { + if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) + { + usage(); + exit(0); + } + if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) + { + puts("pg_checksums (PostgreSQL) " PG_VERSION); + exit(0); + } + } + + while ((c = getopt_long(argc, argv, "D:r:v", long_options, &option_index)) != -1) + { + switch (c) + { + case 'v': + verbose = true; + break; + case 'D': + DataDir = optarg; + break; + case 'r': + if (atoi(optarg) == 0) + { + fprintf(stderr, _("%s: invalid relfilenode specification, must be numeric: %s\n"), progname, optarg); + exit(1); + } + only_relfilenode = pstrdup(optarg); + break; + default: + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + } + + if (DataDir == NULL) + { + if (optind < argc) + DataDir = argv[optind++]; + else + DataDir = getenv("PGDATA"); + + /* If no DataDir was specified, and none could be found, error out */ + if (DataDir == NULL) + { + fprintf(stderr, _("%s: no data directory specified\n"), progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + } + + /* Complain if any arguments remain */ + if (optind < argc) + { + fprintf(stderr, _("%s: too many command-line arguments (first is \"%s\")\n"), + progname, argv[optind]); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit(1); + } + + /* Check if cluster is running */ + ControlFile = get_controlfile(DataDir, progname, &crc_ok); + if (!crc_ok) + { + fprintf(stderr, _("%s: pg_control CRC value is incorrect\n"), progname); + exit(1); + } + + if (ControlFile->pg_control_version != PG_CONTROL_VERSION) + { + fprintf(stderr, _("%s: cluster is not compatible with this version of pg_checksums\n"), + progname); + exit(1); + } + + if (ControlFile->state != DB_SHUTDOWNED && + ControlFile->state != DB_SHUTDOWNED_IN_RECOVERY) + { + fprintf(stderr, _("%s: cluster must be shut down to verify checksums\n"), progname); + exit(1); + } + + if (ControlFile->data_checksum_version == 0) + { + fprintf(stderr, _("%s: data checksums are not enabled in cluster\n"), progname); + exit(1); + } + + /* Scan all files */ + scan_directory(DataDir, "global"); + scan_directory(DataDir, "base"); + scan_directory(DataDir, "pg_tblspc"); + + printf(_("Checksum scan completed\n")); + printf(_("Data checksum version: %d\n"), ControlFile->data_checksum_version); + printf(_("Files scanned: %s\n"), psprintf(INT64_FORMAT, files)); + printf(_("Blocks scanned: %s\n"), psprintf(INT64_FORMAT, blocks)); + printf(_("Bad checksums: %s\n"), psprintf(INT64_FORMAT, badblocks)); + + if (badblocks > 0) + return 1; + + return 0; +} diff --git a/src/bin/pg_checksums/t/001_basic.pl b/src/bin/pg_checksums/t/001_basic.pl new file mode 100644 index 00000000000..4334c806061 --- /dev/null +++ b/src/bin/pg_checksums/t/001_basic.pl @@ -0,0 +1,8 @@ +use strict; +use warnings; +use TestLib; +use Test::More tests => 8; + +program_help_ok('pg_checksums'); +program_version_ok('pg_checksums'); +program_options_handling_ok('pg_checksums'); diff --git a/src/bin/pg_checksums/t/002_actions.pl b/src/bin/pg_checksums/t/002_actions.pl new file mode 100644 index 00000000000..97284e8930c --- /dev/null +++ b/src/bin/pg_checksums/t/002_actions.pl @@ -0,0 +1,160 @@ +# Do basic sanity checks supported by pg_checksums using +# an initialized cluster. + +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 45; + + +# Utility routine to create and check a table with corrupted checksums +# on a wanted tablespace. Note that this stops and starts the node +# multiple times to perform the checks, leaving the node started +# at the end. +sub check_relation_corruption +{ + my $node = shift; + my $table = shift; + my $tablespace = shift; + my $pgdata = $node->data_dir; + + $node->safe_psql('postgres', + "SELECT a INTO $table FROM generate_series(1,10000) AS a; + ALTER TABLE $table SET (autovacuum_enabled=false);"); + + $node->safe_psql('postgres', + "ALTER TABLE ".$table." SET TABLESPACE ".$tablespace.";"); + + my $file_corrupted = $node->safe_psql('postgres', + "SELECT pg_relation_filepath('$table');"); + my $relfilenode_corrupted = $node->safe_psql('postgres', + "SELECT relfilenode FROM pg_class WHERE relname = '$table';"); + + # Set page header and block size + my $pageheader_size = 24; + my $block_size = $node->safe_psql('postgres', 'SHOW block_size;'); + $node->stop; + + # Checksums are correct for single relfilenode as the table is not + # corrupted yet. + command_ok(['pg_checksums', '-D', $pgdata, + '-r', $relfilenode_corrupted], + "succeeds for single relfilenode on tablespace $tablespace with offline cluster"); + + # Time to create some corruption + open my $file, '+<', "$pgdata/$file_corrupted"; + seek($file, $pageheader_size, 0); + syswrite($file, "\0\0\0\0\0\0\0\0\0"); + close $file; + + # Checksum checks on single relfilenode fail + $node->command_checks_all([ 'pg_checksums', '-D', $pgdata, '-r', + $relfilenode_corrupted], + 1, + [qr/Bad checksums:.*1/], + [qr/checksum verification failed/], + "fails with corrupted data for single relfilenode on tablespace $tablespace"); + + # Global checksum checks fail as well + $node->command_checks_all([ 'pg_checksums', '-D', $pgdata], + 1, + [qr/Bad checksums:.*1/], + [qr/checksum verification failed/], + "fails with corrupted data on tablespace $tablespace"); + + # Drop corrupted table again and make sure there is no more corruption. + $node->start; + $node->safe_psql('postgres', "DROP TABLE $table;"); + $node->stop; + $node->command_ok(['pg_checksums', '-D', $pgdata], + "succeeds again after table drop on tablespace $tablespace"); + + $node->start; + return; +} + +# Initialize node with checksums enabled. +my $node = get_new_node('node_checksum'); +$node->init(extra => ['--data-checksums']); +my $pgdata = $node->data_dir; + +# Control file should know that checksums are enabled. +command_like(['pg_controldata', $pgdata], + qr/Data page checksum version:.*1/, + 'checksums enabled in control file'); + +# These are correct but empty files, so they should pass through. +append_to_file "$pgdata/global/99999", ""; +append_to_file "$pgdata/global/99999.123", ""; +append_to_file "$pgdata/global/99999_fsm", ""; +append_to_file "$pgdata/global/99999_init", ""; +append_to_file "$pgdata/global/99999_vm", ""; +append_to_file "$pgdata/global/99999_init.123", ""; +append_to_file "$pgdata/global/99999_fsm.123", ""; +append_to_file "$pgdata/global/99999_vm.123", ""; + +# These are temporary files and folders with dummy contents, which +# should be ignored by the scan. +append_to_file "$pgdata/global/pgsql_tmp_123", "foo"; +mkdir "$pgdata/global/pgsql_tmp"; +append_to_file "$pgdata/global/pgsql_tmp/1.1", "foo"; + +# Checksums pass on a newly-created cluster +command_ok(['pg_checksums', '-D', $pgdata], + "succeeds with offline cluster"); + +# Checks cannot happen with an online cluster +$node->start; +command_fails(['pg_checksums', '-D', $pgdata], + "fails with online cluster"); + +# Check corruption of table on default tablespace. +check_relation_corruption($node, 'corrupt1', 'pg_default'); + +# Create tablespace to check corruptions in a non-default tablespace. +my $basedir = $node->basedir; +my $tablespace_dir = "$basedir/ts_corrupt_dir"; +mkdir ($tablespace_dir); +$tablespace_dir = TestLib::real_dir($tablespace_dir); +$node->safe_psql('postgres', + "CREATE TABLESPACE ts_corrupt LOCATION '$tablespace_dir';"); +check_relation_corruption($node, 'corrupt2', 'ts_corrupt'); + +# Utility routine to check that pg_checksums is able to detect +# correctly-named relation files filled with some corrupted data. +sub fail_corrupt +{ + my $node = shift; + my $file = shift; + my $pgdata = $node->data_dir; + + # Create the file with some dummy data in it. + my $file_name = "$pgdata/global/$file"; + append_to_file $file_name, "foo"; + + $node->command_checks_all([ 'pg_checksums', '-D', $pgdata], + 1, + [qr/^$/], + [qr/could not read block 0 in file.*$file\":/], + "fails for corrupted data in $file"); + + # Remove file to prevent future lookup errors on conflicts. + unlink $file_name; + return; +} + +# Stop instance for the follow-up checks. +$node->stop; + +# Authorized relation files filled with corrupted data cause the +# checksum checks to fail. Make sure to use file names different +# than the previous ones. +fail_corrupt($node, "99990"); +fail_corrupt($node, "99990.123"); +fail_corrupt($node, "99990_fsm"); +fail_corrupt($node, "99990_init"); +fail_corrupt($node, "99990_vm"); +fail_corrupt($node, "99990_init.123"); +fail_corrupt($node, "99990_fsm.123"); +fail_corrupt($node, "99990_vm.123"); diff --git a/src/bin/pg_verify_checksums/.gitignore b/src/bin/pg_verify_checksums/.gitignore deleted file mode 100644 index 0e5e569a54c..00000000000 --- a/src/bin/pg_verify_checksums/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/pg_verify_checksums - -/tmp_check/ diff --git a/src/bin/pg_verify_checksums/Makefile b/src/bin/pg_verify_checksums/Makefile deleted file mode 100644 index ab6d3ea9e2e..00000000000 --- a/src/bin/pg_verify_checksums/Makefile +++ /dev/null @@ -1,42 +0,0 @@ -#------------------------------------------------------------------------- -# -# Makefile for src/bin/pg_verify_checksums -# -# Copyright (c) 1998-2019, PostgreSQL Global Development Group -# -# src/bin/pg_verify_checksums/Makefile -# -#------------------------------------------------------------------------- - -PGFILEDESC = "pg_verify_checksums - verify data checksums in an offline cluster" -PGAPPICON=win32 - -subdir = src/bin/pg_verify_checksums -top_builddir = ../../.. -include $(top_builddir)/src/Makefile.global - -OBJS= pg_verify_checksums.o $(WIN32RES) - -all: pg_verify_checksums - -pg_verify_checksums: $(OBJS) | submake-libpgport - $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) - -install: all installdirs - $(INSTALL_PROGRAM) pg_verify_checksums$(X) '$(DESTDIR)$(bindir)/pg_verify_checksums$(X)' - -installdirs: - $(MKDIR_P) '$(DESTDIR)$(bindir)' - -uninstall: - rm -f '$(DESTDIR)$(bindir)/pg_verify_checksums$(X)' - -clean distclean maintainer-clean: - rm -f pg_verify_checksums$(X) $(OBJS) - rm -rf tmp_check - -check: - $(prove_check) - -installcheck: - $(prove_installcheck) diff --git a/src/bin/pg_verify_checksums/nls.mk b/src/bin/pg_verify_checksums/nls.mk deleted file mode 100644 index 893efaf0f01..00000000000 --- a/src/bin/pg_verify_checksums/nls.mk +++ /dev/null @@ -1,4 +0,0 @@ -# src/bin/pg_verify_checksums/nls.mk -CATALOG_NAME = pg_verify_checksums -AVAIL_LANGUAGES = -GETTEXT_FILES = pg_verify_checksums.c diff --git a/src/bin/pg_verify_checksums/pg_verify_checksums.c b/src/bin/pg_verify_checksums/pg_verify_checksums.c deleted file mode 100644 index 4c7c055b314..00000000000 --- a/src/bin/pg_verify_checksums/pg_verify_checksums.c +++ /dev/null @@ -1,354 +0,0 @@ -/* - * pg_verify_checksums - * - * Verifies page level checksums in an offline cluster - * - * Copyright (c) 2010-2019, PostgreSQL Global Development Group - * - * src/bin/pg_verify_checksums/pg_verify_checksums.c - */ -#include "postgres_fe.h" - -#include -#include -#include - -#include "catalog/pg_control.h" -#include "common/controldata_utils.h" -#include "getopt_long.h" -#include "pg_getopt.h" -#include "storage/bufpage.h" -#include "storage/checksum.h" -#include "storage/checksum_impl.h" -#include "storage/fd.h" - - -static int64 files = 0; -static int64 blocks = 0; -static int64 badblocks = 0; -static ControlFileData *ControlFile; - -static char *only_relfilenode = NULL; -static bool verbose = false; - -static const char *progname; - -static void -usage(void) -{ - printf(_("%s verifies data checksums in a PostgreSQL database cluster.\n\n"), progname); - printf(_("Usage:\n")); - printf(_(" %s [OPTION]... [DATADIR]\n"), progname); - printf(_("\nOptions:\n")); - printf(_(" [-D, --pgdata=]DATADIR data directory\n")); - printf(_(" -v, --verbose output verbose messages\n")); - printf(_(" -r RELFILENODE check only relation with specified relfilenode\n")); - printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -?, --help show this help, then exit\n")); - printf(_("\nIf no data directory (DATADIR) is specified, " - "the environment variable PGDATA\nis used.\n\n")); - printf(_("Report bugs to .\n")); -} - -/* - * List of files excluded from checksum validation. - * - * Note: this list should be kept in sync with what basebackup.c includes. - */ -static const char *const skip[] = { - "pg_control", - "pg_filenode.map", - "pg_internal.init", - "PG_VERSION", -#ifdef EXEC_BACKEND - "config_exec_params", - "config_exec_params.new", -#endif - NULL, -}; - -static bool -skipfile(const char *fn) -{ - const char *const *f; - - for (f = skip; *f; f++) - if (strcmp(*f, fn) == 0) - return true; - - return false; -} - -static void -scan_file(const char *fn, BlockNumber segmentno) -{ - PGAlignedBlock buf; - PageHeader header = (PageHeader) buf.data; - int f; - BlockNumber blockno; - - f = open(fn, O_RDONLY | PG_BINARY, 0); - if (f < 0) - { - fprintf(stderr, _("%s: could not open file \"%s\": %s\n"), - progname, fn, strerror(errno)); - exit(1); - } - - files++; - - for (blockno = 0;; blockno++) - { - uint16 csum; - int r = read(f, buf.data, BLCKSZ); - - if (r == 0) - break; - if (r != BLCKSZ) - { - fprintf(stderr, _("%s: could not read block %u in file \"%s\": read %d of %d\n"), - progname, blockno, fn, r, BLCKSZ); - exit(1); - } - blocks++; - - /* New pages have no checksum yet */ - if (PageIsNew(header)) - continue; - - csum = pg_checksum_page(buf.data, blockno + segmentno * RELSEG_SIZE); - if (csum != header->pd_checksum) - { - if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION) - fprintf(stderr, _("%s: checksum verification failed in file \"%s\", block %u: calculated checksum %X but block contains %X\n"), - progname, fn, blockno, csum, header->pd_checksum); - badblocks++; - } - } - - if (verbose) - fprintf(stderr, - _("%s: checksums verified in file \"%s\"\n"), progname, fn); - - close(f); -} - -static void -scan_directory(const char *basedir, const char *subdir) -{ - char path[MAXPGPATH]; - DIR *dir; - struct dirent *de; - - snprintf(path, sizeof(path), "%s/%s", basedir, subdir); - dir = opendir(path); - if (!dir) - { - fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), - progname, path, strerror(errno)); - exit(1); - } - while ((de = readdir(dir)) != NULL) - { - char fn[MAXPGPATH]; - struct stat st; - - if (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0) - continue; - - /* Skip temporary files */ - if (strncmp(de->d_name, - PG_TEMP_FILE_PREFIX, - strlen(PG_TEMP_FILE_PREFIX)) == 0) - continue; - - /* Skip temporary folders */ - if (strncmp(de->d_name, - PG_TEMP_FILES_DIR, - strlen(PG_TEMP_FILES_DIR)) == 0) - return; - - snprintf(fn, sizeof(fn), "%s/%s", path, de->d_name); - if (lstat(fn, &st) < 0) - { - fprintf(stderr, _("%s: could not stat file \"%s\": %s\n"), - progname, fn, strerror(errno)); - exit(1); - } - if (S_ISREG(st.st_mode)) - { - char fnonly[MAXPGPATH]; - char *forkpath, - *segmentpath; - BlockNumber segmentno = 0; - - if (skipfile(de->d_name)) - continue; - - /* - * Cut off at the segment boundary (".") to get the segment number - * in order to mix it into the checksum. Then also cut off at the - * fork boundary, to get the relfilenode the file belongs to for - * filtering. - */ - strlcpy(fnonly, de->d_name, sizeof(fnonly)); - segmentpath = strchr(fnonly, '.'); - if (segmentpath != NULL) - { - *segmentpath++ = '\0'; - segmentno = atoi(segmentpath); - if (segmentno == 0) - { - fprintf(stderr, _("%s: invalid segment number %d in file name \"%s\"\n"), - progname, segmentno, fn); - exit(1); - } - } - - forkpath = strchr(fnonly, '_'); - if (forkpath != NULL) - *forkpath++ = '\0'; - - if (only_relfilenode && strcmp(only_relfilenode, fnonly) != 0) - /* Relfilenode not to be included */ - continue; - - scan_file(fn, segmentno); - } -#ifndef WIN32 - else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)) -#else - else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn)) -#endif - scan_directory(path, de->d_name); - } - closedir(dir); -} - -int -main(int argc, char *argv[]) -{ - static struct option long_options[] = { - {"pgdata", required_argument, NULL, 'D'}, - {"verbose", no_argument, NULL, 'v'}, - {NULL, 0, NULL, 0} - }; - - char *DataDir = NULL; - int c; - int option_index; - bool crc_ok; - - set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verify_checksums")); - - progname = get_progname(argv[0]); - - if (argc > 1) - { - if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) - { - usage(); - exit(0); - } - if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) - { - puts("pg_verify_checksums (PostgreSQL) " PG_VERSION); - exit(0); - } - } - - while ((c = getopt_long(argc, argv, "D:r:v", long_options, &option_index)) != -1) - { - switch (c) - { - case 'v': - verbose = true; - break; - case 'D': - DataDir = optarg; - break; - case 'r': - if (atoi(optarg) == 0) - { - fprintf(stderr, _("%s: invalid relfilenode specification, must be numeric: %s\n"), progname, optarg); - exit(1); - } - only_relfilenode = pstrdup(optarg); - break; - default: - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); - exit(1); - } - } - - if (DataDir == NULL) - { - if (optind < argc) - DataDir = argv[optind++]; - else - DataDir = getenv("PGDATA"); - - /* If no DataDir was specified, and none could be found, error out */ - if (DataDir == NULL) - { - fprintf(stderr, _("%s: no data directory specified\n"), progname); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); - exit(1); - } - } - - /* Complain if any arguments remain */ - if (optind < argc) - { - fprintf(stderr, _("%s: too many command-line arguments (first is \"%s\")\n"), - progname, argv[optind]); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), - progname); - exit(1); - } - - /* Check if cluster is running */ - ControlFile = get_controlfile(DataDir, progname, &crc_ok); - if (!crc_ok) - { - fprintf(stderr, _("%s: pg_control CRC value is incorrect\n"), progname); - exit(1); - } - - if (ControlFile->pg_control_version != PG_CONTROL_VERSION) - { - fprintf(stderr, _("%s: cluster is not compatible with this version of pg_verify_checksums\n"), - progname); - exit(1); - } - - if (ControlFile->state != DB_SHUTDOWNED && - ControlFile->state != DB_SHUTDOWNED_IN_RECOVERY) - { - fprintf(stderr, _("%s: cluster must be shut down to verify checksums\n"), progname); - exit(1); - } - - if (ControlFile->data_checksum_version == 0) - { - fprintf(stderr, _("%s: data checksums are not enabled in cluster\n"), progname); - exit(1); - } - - /* Scan all files */ - scan_directory(DataDir, "global"); - scan_directory(DataDir, "base"); - scan_directory(DataDir, "pg_tblspc"); - - printf(_("Checksum scan completed\n")); - printf(_("Data checksum version: %d\n"), ControlFile->data_checksum_version); - printf(_("Files scanned: %s\n"), psprintf(INT64_FORMAT, files)); - printf(_("Blocks scanned: %s\n"), psprintf(INT64_FORMAT, blocks)); - printf(_("Bad checksums: %s\n"), psprintf(INT64_FORMAT, badblocks)); - - if (badblocks > 0) - return 1; - - return 0; -} diff --git a/src/bin/pg_verify_checksums/t/001_basic.pl b/src/bin/pg_verify_checksums/t/001_basic.pl deleted file mode 100644 index 1fa2e12db26..00000000000 --- a/src/bin/pg_verify_checksums/t/001_basic.pl +++ /dev/null @@ -1,8 +0,0 @@ -use strict; -use warnings; -use TestLib; -use Test::More tests => 8; - -program_help_ok('pg_verify_checksums'); -program_version_ok('pg_verify_checksums'); -program_options_handling_ok('pg_verify_checksums'); diff --git a/src/bin/pg_verify_checksums/t/002_actions.pl b/src/bin/pg_verify_checksums/t/002_actions.pl deleted file mode 100644 index 74ad5ad7235..00000000000 --- a/src/bin/pg_verify_checksums/t/002_actions.pl +++ /dev/null @@ -1,160 +0,0 @@ -# Do basic sanity checks supported by pg_verify_checksums using -# an initialized cluster. - -use strict; -use warnings; -use PostgresNode; -use TestLib; -use Test::More tests => 45; - - -# Utility routine to create and check a table with corrupted checksums -# on a wanted tablespace. Note that this stops and starts the node -# multiple times to perform the checks, leaving the node started -# at the end. -sub check_relation_corruption -{ - my $node = shift; - my $table = shift; - my $tablespace = shift; - my $pgdata = $node->data_dir; - - $node->safe_psql('postgres', - "SELECT a INTO $table FROM generate_series(1,10000) AS a; - ALTER TABLE $table SET (autovacuum_enabled=false);"); - - $node->safe_psql('postgres', - "ALTER TABLE ".$table." SET TABLESPACE ".$tablespace.";"); - - my $file_corrupted = $node->safe_psql('postgres', - "SELECT pg_relation_filepath('$table');"); - my $relfilenode_corrupted = $node->safe_psql('postgres', - "SELECT relfilenode FROM pg_class WHERE relname = '$table';"); - - # Set page header and block size - my $pageheader_size = 24; - my $block_size = $node->safe_psql('postgres', 'SHOW block_size;'); - $node->stop; - - # Checksums are correct for single relfilenode as the table is not - # corrupted yet. - command_ok(['pg_verify_checksums', '-D', $pgdata, - '-r', $relfilenode_corrupted], - "succeeds for single relfilenode on tablespace $tablespace with offline cluster"); - - # Time to create some corruption - open my $file, '+<', "$pgdata/$file_corrupted"; - seek($file, $pageheader_size, 0); - syswrite($file, "\0\0\0\0\0\0\0\0\0"); - close $file; - - # Checksum checks on single relfilenode fail - $node->command_checks_all([ 'pg_verify_checksums', '-D', $pgdata, '-r', - $relfilenode_corrupted], - 1, - [qr/Bad checksums:.*1/], - [qr/checksum verification failed/], - "fails with corrupted data for single relfilenode on tablespace $tablespace"); - - # Global checksum checks fail as well - $node->command_checks_all([ 'pg_verify_checksums', '-D', $pgdata], - 1, - [qr/Bad checksums:.*1/], - [qr/checksum verification failed/], - "fails with corrupted data on tablespace $tablespace"); - - # Drop corrupted table again and make sure there is no more corruption. - $node->start; - $node->safe_psql('postgres', "DROP TABLE $table;"); - $node->stop; - $node->command_ok(['pg_verify_checksums', '-D', $pgdata], - "succeeds again after table drop on tablespace $tablespace"); - - $node->start; - return; -} - -# Initialize node with checksums enabled. -my $node = get_new_node('node_checksum'); -$node->init(extra => ['--data-checksums']); -my $pgdata = $node->data_dir; - -# Control file should know that checksums are enabled. -command_like(['pg_controldata', $pgdata], - qr/Data page checksum version:.*1/, - 'checksums enabled in control file'); - -# These are correct but empty files, so they should pass through. -append_to_file "$pgdata/global/99999", ""; -append_to_file "$pgdata/global/99999.123", ""; -append_to_file "$pgdata/global/99999_fsm", ""; -append_to_file "$pgdata/global/99999_init", ""; -append_to_file "$pgdata/global/99999_vm", ""; -append_to_file "$pgdata/global/99999_init.123", ""; -append_to_file "$pgdata/global/99999_fsm.123", ""; -append_to_file "$pgdata/global/99999_vm.123", ""; - -# These are temporary files and folders with dummy contents, which -# should be ignored by the scan. -append_to_file "$pgdata/global/pgsql_tmp_123", "foo"; -mkdir "$pgdata/global/pgsql_tmp"; -append_to_file "$pgdata/global/pgsql_tmp/1.1", "foo"; - -# Checksums pass on a newly-created cluster -command_ok(['pg_verify_checksums', '-D', $pgdata], - "succeeds with offline cluster"); - -# Checks cannot happen with an online cluster -$node->start; -command_fails(['pg_verify_checksums', '-D', $pgdata], - "fails with online cluster"); - -# Check corruption of table on default tablespace. -check_relation_corruption($node, 'corrupt1', 'pg_default'); - -# Create tablespace to check corruptions in a non-default tablespace. -my $basedir = $node->basedir; -my $tablespace_dir = "$basedir/ts_corrupt_dir"; -mkdir ($tablespace_dir); -$tablespace_dir = TestLib::real_dir($tablespace_dir); -$node->safe_psql('postgres', - "CREATE TABLESPACE ts_corrupt LOCATION '$tablespace_dir';"); -check_relation_corruption($node, 'corrupt2', 'ts_corrupt'); - -# Utility routine to check that pg_verify_checksums is able to detect -# correctly-named relation files filled with some corrupted data. -sub fail_corrupt -{ - my $node = shift; - my $file = shift; - my $pgdata = $node->data_dir; - - # Create the file with some dummy data in it. - my $file_name = "$pgdata/global/$file"; - append_to_file $file_name, "foo"; - - $node->command_checks_all([ 'pg_verify_checksums', '-D', $pgdata], - 1, - [qr/^$/], - [qr/could not read block 0 in file.*$file\":/], - "fails for corrupted data in $file"); - - # Remove file to prevent future lookup errors on conflicts. - unlink $file_name; - return; -} - -# Stop instance for the follow-up checks. -$node->stop; - -# Authorized relation files filled with corrupted data cause the -# checksum checks to fail. Make sure to use file names different -# than the previous ones. -fail_corrupt($node, "99990"); -fail_corrupt($node, "99990.123"); -fail_corrupt($node, "99990_fsm"); -fail_corrupt($node, "99990_init"); -fail_corrupt($node, "99990_vm"); -fail_corrupt($node, "99990_init.123"); -fail_corrupt($node, "99990_fsm.123"); -fail_corrupt($node, "99990_vm.123"); -- cgit v1.2.3