diff options
| -rw-r--r-- | src/backend/replication/Makefile | 1 | ||||
| -rw-r--r-- | src/backend/replication/backup_manifest.c | 375 | ||||
| -rw-r--r-- | src/backend/replication/basebackup.c | 391 | ||||
| -rw-r--r-- | src/include/replication/backup_manifest.h | 51 |
4 files changed, 429 insertions, 389 deletions
diff --git a/src/backend/replication/Makefile b/src/backend/replication/Makefile index fd08f093e1d..a0381e52f31 100644 --- a/src/backend/replication/Makefile +++ b/src/backend/replication/Makefile @@ -15,6 +15,7 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS) OBJS = \ + backup_manifest.o \ basebackup.o \ repl_gram.o \ slot.o \ diff --git a/src/backend/replication/backup_manifest.c b/src/backend/replication/backup_manifest.c new file mode 100644 index 00000000000..8aead70726a --- /dev/null +++ b/src/backend/replication/backup_manifest.c @@ -0,0 +1,375 @@ +/*------------------------------------------------------------------------- + * + * backup_manifest.c + * code for generating and sending a backup manifest + * + * Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/backup_manifest.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/timeline.h" +#include "libpq/libpq.h" +#include "libpq/pqformat.h" +#include "mb/pg_wchar.h" +#include "replication/backup_manifest.h" +#include "utils/builtins.h" +#include "utils/json.h" + +/* + * Does the user want a backup manifest? + * + * It's simplest to always have a manifest_info object, so that we don't need + * checks for NULL pointers in too many places. However, if the user doesn't + * want a manifest, we set manifest->buffile to NULL. + */ +static inline bool +IsManifestEnabled(manifest_info *manifest) +{ + return (manifest->buffile != NULL); +} + +/* + * Convenience macro for appending data to the backup manifest. + */ +#define AppendToManifest(manifest, ...) \ + { \ + char *_manifest_s = psprintf(__VA_ARGS__); \ + AppendStringToManifest(manifest, _manifest_s); \ + pfree(_manifest_s); \ + } + +/* + * Initialize state so that we can construct a backup manifest. + * + * NB: Although the checksum type for the data files is configurable, the + * checksum for the manifest itself always uses SHA-256. See comments in + * SendBackupManifest. + */ +void +InitializeManifest(manifest_info *manifest, manifest_option want_manifest, + pg_checksum_type manifest_checksum_type) +{ + if (want_manifest == MANIFEST_OPTION_NO) + manifest->buffile = NULL; + else + manifest->buffile = BufFileCreateTemp(false); + manifest->checksum_type = manifest_checksum_type; + pg_sha256_init(&manifest->manifest_ctx); + manifest->manifest_size = UINT64CONST(0); + manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE); + manifest->first_file = true; + manifest->still_checksumming = true; + + if (want_manifest != MANIFEST_OPTION_NO) + AppendToManifest(manifest, + "{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n" + "\"Files\": ["); +} + +/* + * Append a cstring to the manifest. + */ +void +AppendStringToManifest(manifest_info *manifest, char *s) +{ + int len = strlen(s); + size_t written; + + Assert(manifest != NULL); + if (manifest->still_checksumming) + pg_sha256_update(&manifest->manifest_ctx, (uint8 *) s, len); + written = BufFileWrite(manifest->buffile, s, len); + if (written != len) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to temporary file: %m"))); + manifest->manifest_size += len; +} + +/* + * Add an entry to the backup manifest for a file. + */ +void +AddFileToManifest(manifest_info *manifest, const char *spcoid, + const char *pathname, size_t size, pg_time_t mtime, + pg_checksum_context *checksum_ctx) +{ + char pathbuf[MAXPGPATH]; + int pathlen; + StringInfoData buf; + + if (!IsManifestEnabled(manifest)) + return; + + /* + * If this file is part of a tablespace, the pathname passed to this + * function will be relative to the tar file that contains it. We want the + * pathname relative to the data directory (ignoring the intermediate + * symlink traversal). + */ + if (spcoid != NULL) + { + snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid, + pathname); + pathname = pathbuf; + } + + /* + * Each file's entry needs to be separated from any entry that follows by a + * comma, but there's no comma before the first one or after the last one. + * To make that work, adding a file to the manifest starts by terminating + * the most recently added line, with a comma if appropriate, but does not + * terminate the line inserted for this file. + */ + initStringInfo(&buf); + if (manifest->first_file) + { + appendStringInfoString(&buf, "\n"); + manifest->first_file = false; + } + else + appendStringInfoString(&buf, ",\n"); + + /* + * Write the relative pathname to this file out to the manifest. The + * manifest is always stored in UTF-8, so we have to encode paths that are + * not valid in that encoding. + */ + pathlen = strlen(pathname); + if (!manifest->force_encode && + pg_verify_mbstr(PG_UTF8, pathname, pathlen, true)) + { + appendStringInfoString(&buf, "{ \"Path\": "); + escape_json(&buf, pathname); + appendStringInfoString(&buf, ", "); + } + else + { + appendStringInfoString(&buf, "{ \"Encoded-Path\": \""); + enlargeStringInfo(&buf, 2 * pathlen); + buf.len += hex_encode((char *) pathname, pathlen, + &buf.data[buf.len]); + appendStringInfoString(&buf, "\", "); + } + + appendStringInfo(&buf, "\"Size\": %zu, ", size); + + /* + * Convert last modification time to a string and append it to the + * manifest. Since it's not clear what time zone to use and since time + * zone definitions can change, possibly causing confusion, use GMT + * always. + */ + appendStringInfoString(&buf, "\"Last-Modified\": \""); + enlargeStringInfo(&buf, 128); + buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z", + pg_gmtime(&mtime)); + appendStringInfoString(&buf, "\""); + + /* Add checksum information. */ + if (checksum_ctx->type != CHECKSUM_TYPE_NONE) + { + uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH]; + int checksumlen; + + checksumlen = pg_checksum_final(checksum_ctx, checksumbuf); + + appendStringInfo(&buf, + ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"", + pg_checksum_type_name(checksum_ctx->type)); + enlargeStringInfo(&buf, 2 * checksumlen); + buf.len += hex_encode((char *) checksumbuf, checksumlen, + &buf.data[buf.len]); + appendStringInfoString(&buf, "\""); + } + + /* Close out the object. */ + appendStringInfoString(&buf, " }"); + + /* OK, add it to the manifest. */ + AppendStringToManifest(manifest, buf.data); + + /* Avoid leaking memory. */ + pfree(buf.data); +} + +/* + * Add information about the WAL that will need to be replayed when restoring + * this backup to the manifest. + */ +void +AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr, + TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli) +{ + List *timelines; + ListCell *lc; + bool first_wal_range = true; + bool found_start_timeline = false; + + if (!IsManifestEnabled(manifest)) + return; + + /* Terminate the list of files. */ + AppendStringToManifest(manifest, "\n],\n"); + + /* Read the timeline history for the ending timeline. */ + timelines = readTimeLineHistory(endtli); + + /* Start a list of LSN ranges. */ + AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n"); + + foreach(lc, timelines) + { + TimeLineHistoryEntry *entry = lfirst(lc); + XLogRecPtr tl_beginptr; + + /* + * We only care about timelines that were active during the backup. + * Skip any that ended before the backup started. (Note that if + * entry->end is InvalidXLogRecPtr, it means that the timeline has not + * yet ended.) + */ + if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr) + continue; + + /* + * Because the timeline history file lists newer timelines before + * older ones, the first timeline we encounter that is new enough to + * matter ought to match the ending timeline of the backup. + */ + if (first_wal_range && endtli != entry->tli) + ereport(ERROR, + errmsg("expected end timeline %u but found timeline %u", + starttli, entry->tli)); + + if (!XLogRecPtrIsInvalid(entry->begin)) + tl_beginptr = entry->begin; + else + { + tl_beginptr = startptr; + + /* + * If we reach a TLI that has no valid beginning LSN, there can't + * be any more timelines in the history after this point, so we'd + * better have arrived at the expected starting TLI. If not, + * something's gone horribly wrong. + */ + if (starttli != entry->tli) + ereport(ERROR, + errmsg("expected start timeline %u but found timeline %u", + starttli, entry->tli)); + } + + AppendToManifest(manifest, + "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }", + first_wal_range ? "" : ",\n", + entry->tli, + (uint32) (tl_beginptr >> 32), (uint32) tl_beginptr, + (uint32) (endptr >> 32), (uint32) endptr); + + if (starttli == entry->tli) + { + found_start_timeline = true; + break; + } + + endptr = entry->begin; + first_wal_range = false; + } + + /* + * The last entry in the timeline history for the ending timeline should + * be the ending timeline itself. Verify that this is what we observed. + */ + if (!found_start_timeline) + ereport(ERROR, + errmsg("start timeline %u not found history of timeline %u", + starttli, endtli)); + + /* Terminate the list of WAL ranges. */ + AppendStringToManifest(manifest, "\n],\n"); +} + +/* + * Finalize the backup manifest, and send it to the client. + */ +void +SendBackupManifest(manifest_info *manifest) +{ + StringInfoData protobuf; + uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH]; + char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH]; + size_t manifest_bytes_done = 0; + + if (!IsManifestEnabled(manifest)) + return; + + /* + * Append manifest checksum, so that the problems with the manifest itself + * can be detected. + * + * We always use SHA-256 for this, regardless of what algorithm is chosen + * for checksumming the files. If we ever want to make the checksum + * algorithm used for the manifest file variable, the client will need a + * way to figure out which algorithm to use as close to the beginning of + * the manifest file as possible, to avoid having to read the whole thing + * twice. + */ + manifest->still_checksumming = false; + pg_sha256_final(&manifest->manifest_ctx, checksumbuf); + AppendStringToManifest(manifest, "\"Manifest-Checksum\": \""); + hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf); + checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0'; + AppendStringToManifest(manifest, checksumstringbuf); + AppendStringToManifest(manifest, "\"}\n"); + + /* + * We've written all the data to the manifest file. Rewind the file so + * that we can read it all back. + */ + if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not rewind temporary file: %m"))); + + /* Send CopyOutResponse message */ + pq_beginmessage(&protobuf, 'H'); + pq_sendbyte(&protobuf, 0); /* overall format */ + pq_sendint16(&protobuf, 0); /* natts */ + pq_endmessage(&protobuf); + + /* + * Send CopyData messages. + * + * We choose to read back the data from the temporary file in chunks of + * size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O + * size, so it seems to make sense to match that value here. + */ + while (manifest_bytes_done < manifest->manifest_size) + { + char manifestbuf[BLCKSZ]; + size_t bytes_to_read; + size_t rc; + + bytes_to_read = Min(sizeof(manifestbuf), + manifest->manifest_size - manifest_bytes_done); + rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read); + if (rc != bytes_to_read) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read from temporary file: %m"))); + pq_putmessage('d', manifestbuf, bytes_to_read); + manifest_bytes_done += bytes_to_read; + } + + /* No more data, so send CopyDone message */ + pq_putemptymessage('c'); + + /* Release resources */ + BufFileClose(manifest->buffile); +} diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index f5b2411d549..f3fb5716a4b 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -16,10 +16,8 @@ #include <unistd.h> #include <time.h> -#include "access/timeline.h" #include "access/xlog_internal.h" /* for pg_start/stop_backup */ #include "catalog/pg_type.h" -#include "common/checksum_helper.h" #include "common/file_perm.h" #include "commands/progress.h" #include "lib/stringinfo.h" @@ -32,9 +30,9 @@ #include "port.h" #include "postmaster/syslogger.h" #include "replication/basebackup.h" +#include "replication/backup_manifest.h" #include "replication/walsender.h" #include "replication/walsender_private.h" -#include "storage/buffile.h" #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" @@ -42,19 +40,11 @@ #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" -#include "utils/json.h" #include "utils/ps_status.h" #include "utils/relcache.h" #include "utils/resowner.h" #include "utils/timestamp.h" -typedef enum manifest_option -{ - MANIFEST_OPTION_YES, - MANIFEST_OPTION_NO, - MANIFEST_OPTION_FORCE_ENCODE -} manifest_option; - typedef struct { const char *label; @@ -68,18 +58,6 @@ typedef struct pg_checksum_type manifest_checksum_type; } basebackup_options; -struct manifest_info -{ - BufFile *buffile; - pg_checksum_type checksum_type; - pg_sha256_ctx manifest_ctx; - uint64 manifest_size; - bool force_encode; - bool first_file; - bool still_checksumming; -}; - - static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, manifest_info *manifest, const char *spcoid); @@ -94,18 +72,6 @@ static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *sta bool sizeonly); static void send_int8_string(StringInfoData *buf, int64 intval); static void SendBackupHeader(List *tablespaces); -static bool IsManifestEnabled(manifest_info *manifest); -static void InitializeManifest(manifest_info *manifest, - basebackup_options *opt); -static void AppendStringToManifest(manifest_info *manifest, char *s); -static void AddFileToManifest(manifest_info *manifest, const char *spcoid, - const char *pathname, size_t size, - pg_time_t mtime, - pg_checksum_context *checksum_ctx); -static void AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr, - TimeLineID starttli, XLogRecPtr endptr, - TimeLineID endtli); -static void SendBackupManifest(manifest_info *manifest); static void perform_base_backup(basebackup_options *opt); static void parse_basebackup_options(List *options, basebackup_options *opt); static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli); @@ -142,16 +108,6 @@ do { \ (errmsg("could not read from file \"%s\"", filename))); \ } while (0) -/* - * Convenience macro for appending data to the backup manifest. - */ -#define AppendToManifest(manifest, ...) \ - { \ - char *_manifest_s = psprintf(__VA_ARGS__); \ - AppendStringToManifest(manifest, _manifest_s); \ - pfree(_manifest_s); \ - } - /* The actual number of bytes, transfer of which may cause sleep. */ static uint64 throttling_sample; @@ -342,7 +298,7 @@ perform_base_backup(basebackup_options *opt) labelfile = makeStringInfo(); tblspc_map_file = makeStringInfo(); - InitializeManifest(&manifest, opt); + InitializeManifest(&manifest, opt->manifest, opt->manifest_checksum_type); total_checksum_failures = 0; @@ -1069,349 +1025,6 @@ SendBackupHeader(List *tablespaces) } /* - * Does the user want a backup manifest? - * - * It's simplest to always have a manifest_info object, so that we don't need - * checks for NULL pointers in too many places. However, if the user doesn't - * want a manifest, we set manifest->buffile to NULL. - */ -static bool -IsManifestEnabled(manifest_info *manifest) -{ - return (manifest->buffile != NULL); -} - -/* - * Initialize state so that we can construct a backup manifest. - * - * NB: Although the checksum type for the data files is configurable, the - * checksum for the manifest itself always uses SHA-256. See comments in - * SendBackupManifest. - */ -static void -InitializeManifest(manifest_info *manifest, basebackup_options *opt) -{ - if (opt->manifest == MANIFEST_OPTION_NO) - manifest->buffile = NULL; - else - manifest->buffile = BufFileCreateTemp(false); - manifest->checksum_type = opt->manifest_checksum_type; - pg_sha256_init(&manifest->manifest_ctx); - manifest->manifest_size = UINT64CONST(0); - manifest->force_encode = (opt->manifest == MANIFEST_OPTION_FORCE_ENCODE); - manifest->first_file = true; - manifest->still_checksumming = true; - - if (opt->manifest != MANIFEST_OPTION_NO) - AppendToManifest(manifest, - "{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n" - "\"Files\": ["); -} - -/* - * Append a cstring to the manifest. - */ -static void -AppendStringToManifest(manifest_info *manifest, char *s) -{ - int len = strlen(s); - size_t written; - - Assert(manifest != NULL); - if (manifest->still_checksumming) - pg_sha256_update(&manifest->manifest_ctx, (uint8 *) s, len); - written = BufFileWrite(manifest->buffile, s, len); - if (written != len) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not write to temporary file: %m"))); - manifest->manifest_size += len; -} - -/* - * Add an entry to the backup manifest for a file. - */ -static void -AddFileToManifest(manifest_info *manifest, const char *spcoid, - const char *pathname, size_t size, pg_time_t mtime, - pg_checksum_context *checksum_ctx) -{ - char pathbuf[MAXPGPATH]; - int pathlen; - StringInfoData buf; - - if (!IsManifestEnabled(manifest)) - return; - - /* - * If this file is part of a tablespace, the pathname passed to this - * function will be relative to the tar file that contains it. We want the - * pathname relative to the data directory (ignoring the intermediate - * symlink traversal). - */ - if (spcoid != NULL) - { - snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid, - pathname); - pathname = pathbuf; - } - - /* - * Each file's entry needs to be separated from any entry that follows by a - * comma, but there's no comma before the first one or after the last one. - * To make that work, adding a file to the manifest starts by terminating - * the most recently added line, with a comma if appropriate, but does not - * terminate the line inserted for this file. - */ - initStringInfo(&buf); - if (manifest->first_file) - { - appendStringInfoString(&buf, "\n"); - manifest->first_file = false; - } - else - appendStringInfoString(&buf, ",\n"); - - /* - * Write the relative pathname to this file out to the manifest. The - * manifest is always stored in UTF-8, so we have to encode paths that are - * not valid in that encoding. - */ - pathlen = strlen(pathname); - if (!manifest->force_encode && - pg_verify_mbstr(PG_UTF8, pathname, pathlen, true)) - { - appendStringInfoString(&buf, "{ \"Path\": "); - escape_json(&buf, pathname); - appendStringInfoString(&buf, ", "); - } - else - { - appendStringInfoString(&buf, "{ \"Encoded-Path\": \""); - enlargeStringInfo(&buf, 2 * pathlen); - buf.len += hex_encode((char *) pathname, pathlen, - &buf.data[buf.len]); - appendStringInfoString(&buf, "\", "); - } - - appendStringInfo(&buf, "\"Size\": %zu, ", size); - - /* - * Convert last modification time to a string and append it to the - * manifest. Since it's not clear what time zone to use and since time - * zone definitions can change, possibly causing confusion, use GMT - * always. - */ - appendStringInfoString(&buf, "\"Last-Modified\": \""); - enlargeStringInfo(&buf, 128); - buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z", - pg_gmtime(&mtime)); - appendStringInfoString(&buf, "\""); - - /* Add checksum information. */ - if (checksum_ctx->type != CHECKSUM_TYPE_NONE) - { - uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH]; - int checksumlen; - - checksumlen = pg_checksum_final(checksum_ctx, checksumbuf); - - appendStringInfo(&buf, - ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"", - pg_checksum_type_name(checksum_ctx->type)); - enlargeStringInfo(&buf, 2 * checksumlen); - buf.len += hex_encode((char *) checksumbuf, checksumlen, - &buf.data[buf.len]); - appendStringInfoString(&buf, "\""); - } - - /* Close out the object. */ - appendStringInfoString(&buf, " }"); - - /* OK, add it to the manifest. */ - AppendStringToManifest(manifest, buf.data); - - /* Avoid leaking memory. */ - pfree(buf.data); -} - -/* - * Add information about the WAL that will need to be replayed when restoring - * this backup to the manifest. - */ -static void -AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr, - TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli) -{ - List *timelines; - ListCell *lc; - bool first_wal_range = true; - bool found_start_timeline = false; - - if (!IsManifestEnabled(manifest)) - return; - - /* Terminate the list of files. */ - AppendStringToManifest(manifest, "\n],\n"); - - /* Read the timeline history for the ending timeline. */ - timelines = readTimeLineHistory(endtli); - - /* Start a list of LSN ranges. */ - AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n"); - - foreach(lc, timelines) - { - TimeLineHistoryEntry *entry = lfirst(lc); - XLogRecPtr tl_beginptr; - - /* - * We only care about timelines that were active during the backup. - * Skip any that ended before the backup started. (Note that if - * entry->end is InvalidXLogRecPtr, it means that the timeline has not - * yet ended.) - */ - if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr) - continue; - - /* - * Because the timeline history file lists newer timelines before - * older ones, the first timeline we encounter that is new enough to - * matter ought to match the ending timeline of the backup. - */ - if (first_wal_range && endtli != entry->tli) - ereport(ERROR, - errmsg("expected end timeline %u but found timeline %u", - starttli, entry->tli)); - - if (!XLogRecPtrIsInvalid(entry->begin)) - tl_beginptr = entry->begin; - else - { - tl_beginptr = startptr; - - /* - * If we reach a TLI that has no valid beginning LSN, there can't - * be any more timelines in the history after this point, so we'd - * better have arrived at the expected starting TLI. If not, - * something's gone horribly wrong. - */ - if (starttli != entry->tli) - ereport(ERROR, - errmsg("expected start timeline %u but found timeline %u", - starttli, entry->tli)); - } - - AppendToManifest(manifest, - "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }", - first_wal_range ? "" : ",\n", - entry->tli, - (uint32) (tl_beginptr >> 32), (uint32) tl_beginptr, - (uint32) (endptr >> 32), (uint32) endptr); - - if (starttli == entry->tli) - { - found_start_timeline = true; - break; - } - - endptr = entry->begin; - first_wal_range = false; - } - - /* - * The last entry in the timeline history for the ending timeline should - * be the ending timeline itself. Verify that this is what we observed. - */ - if (!found_start_timeline) - ereport(ERROR, - errmsg("start timeline %u not found history of timeline %u", - starttli, endtli)); - - /* Terminate the list of WAL ranges. */ - AppendStringToManifest(manifest, "\n],\n"); -} - -/* - * Finalize the backup manifest, and send it to the client. - */ -static void -SendBackupManifest(manifest_info *manifest) -{ - StringInfoData protobuf; - uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH]; - char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH]; - size_t manifest_bytes_done = 0; - - if (!IsManifestEnabled(manifest)) - return; - - /* - * Append manifest checksum, so that the problems with the manifest itself - * can be detected. - * - * We always use SHA-256 for this, regardless of what algorithm is chosen - * for checksumming the files. If we ever want to make the checksum - * algorithm used for the manifest file variable, the client will need a - * way to figure out which algorithm to use as close to the beginning of - * the manifest file as possible, to avoid having to read the whole thing - * twice. - */ - manifest->still_checksumming = false; - pg_sha256_final(&manifest->manifest_ctx, checksumbuf); - AppendStringToManifest(manifest, "\"Manifest-Checksum\": \""); - hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf); - checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0'; - AppendStringToManifest(manifest, checksumstringbuf); - AppendStringToManifest(manifest, "\"}\n"); - - /* - * We've written all the data to the manifest file. Rewind the file so - * that we can read it all back. - */ - if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET)) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not rewind temporary file: %m"))); - - /* Send CopyOutResponse message */ - pq_beginmessage(&protobuf, 'H'); - pq_sendbyte(&protobuf, 0); /* overall format */ - pq_sendint16(&protobuf, 0); /* natts */ - pq_endmessage(&protobuf); - - /* - * Send CopyData messages. - * - * We choose to read back the data from the temporary file in chunks of - * size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O - * size, so it seems to make sense to match that value here. - */ - while (manifest_bytes_done < manifest->manifest_size) - { - char manifestbuf[BLCKSZ]; - size_t bytes_to_read; - size_t rc; - - bytes_to_read = Min(sizeof(manifestbuf), - manifest->manifest_size - manifest_bytes_done); - rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read); - if (rc != bytes_to_read) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not read from temporary file: %m"))); - pq_putmessage('d', manifestbuf, bytes_to_read); - manifest_bytes_done += bytes_to_read; - } - - /* No more data, so send CopyDone message */ - pq_putemptymessage('c'); - - /* Release resources */ - BufFileClose(manifest->buffile); -} - -/* * Send a single resultset containing just a single * XLogRecPtr record (in text format) */ diff --git a/src/include/replication/backup_manifest.h b/src/include/replication/backup_manifest.h new file mode 100644 index 00000000000..e7fccddd0da --- /dev/null +++ b/src/include/replication/backup_manifest.h @@ -0,0 +1,51 @@ +/*------------------------------------------------------------------------- + * + * backup_manifest.h + * Routines for generating a backup manifest. + * + * Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group + * + * src/include/replication/backup_manifest.h + * + *------------------------------------------------------------------------- + */ +#ifndef BACKUP_MANIFEST_H +#define BACKUP_MANIFEST_H + +#include "access/xlogdefs.h" +#include "common/checksum_helper.h" +#include "pgtime.h" +#include "storage/buffile.h" + +typedef enum manifest_option +{ + MANIFEST_OPTION_YES, + MANIFEST_OPTION_NO, + MANIFEST_OPTION_FORCE_ENCODE +} manifest_option; + +typedef struct manifest_info +{ + BufFile *buffile; + pg_checksum_type checksum_type; + pg_sha256_ctx manifest_ctx; + uint64 manifest_size; + bool force_encode; + bool first_file; + bool still_checksumming; +} manifest_info; + +extern void InitializeManifest(manifest_info *manifest, + manifest_option want_manifest, + pg_checksum_type manifest_checksum_type); +extern void AppendStringToManifest(manifest_info *manifest, char *s); +extern void AddFileToManifest(manifest_info *manifest, const char *spcoid, + const char *pathname, size_t size, + pg_time_t mtime, + pg_checksum_context *checksum_ctx); +extern void AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr, + TimeLineID starttli, XLogRecPtr endptr, + TimeLineID endtli); +extern void SendBackupManifest(manifest_info *manifest); + +#endif |