Skip to content

Commit c530d28

Browse files
alubennikovaalubennikova
committed
change datafile validation algorithm:
 - validate file block by block by default, not only in case of file-level checksum corruption; - add an option: --skip-block-validation to disable this behaviour; - calculate file checksum at the same time as validate blocks;
1 parent a0ec849 commit c530d28

File tree

6 files changed

+93
-52
lines changed

6 files changed

+93
-52
lines changed

‎src/data.c‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1601,12 +1601,14 @@ validate_one_page(Page page, pgFile *file,
16011601

16021602
/* Valiate pages of datafile in backup one by one */
16031603
bool
1604-
check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
1605-
uint32 backup_version)
1604+
check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
1605+
uint32 checksum_version, uint32 backup_version)
16061606
{
16071607
size_t read_len = 0;
16081608
bool is_valid = true;
16091609
FILE *in;
1610+
pg_crc32 crc;
1611+
bool use_crc32c = (backup_version <= 20021);
16101612

16111613
elog(VERBOSE, "validate relation blocks for file %s", file->name);
16121614

@@ -1623,6 +1625,9 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
16231625
file->path, strerror(errno));
16241626
}
16251627

1628+
/* calc CRC of backup file */
1629+
INIT_FILE_CRC32(use_crc32c, crc);
1630+
16261631
/* read and validate pages one by one */
16271632
while (true)
16281633
{
@@ -1647,6 +1652,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
16471652
blknum, file->path, strerror(errno_tmp));
16481653
}
16491654

1655+
COMP_FILE_CRC32(use_crc32c, crc, &header, read_len);
1656+
16501657
if (header.block < blknum)
16511658
elog(ERROR, "backup is broken at file->path %s block %u",
16521659
file->path, blknum);
@@ -1668,6 +1675,8 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
16681675
elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d",
16691676
blknum, file->path, read_len, header.compressed_size);
16701677

1678+
COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len);
1679+
16711680
if (header.compressed_size != BLCKSZ
16721681
|| page_may_be_compressed(compressed_page.data, file->compress_alg,
16731682
backup_version))
@@ -1706,5 +1715,14 @@ check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
17061715
}
17071716
}
17081717

1718+
FIN_FILE_CRC32(use_crc32c, crc);
1719+
1720+
if (crc != file->crc)
1721+
{
1722+
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
1723+
file->path, file->crc, crc);
1724+
is_valid = false;
1725+
}
1726+
17091727
return is_valid;
17101728
}

‎src/dir.c‎

Lines changed: 4 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -267,49 +267,27 @@ pgFileGetCRC(const char *file_path, bool use_crc32c)
267267
size_t len;
268268
int errno_tmp;
269269

270-
#define INIT_FILE_CRC32(crc) \
271-
do { \
272-
if (use_crc32c) \
273-
INIT_CRC32C(crc); \
274-
else \
275-
INIT_TRADITIONAL_CRC32(crc); \
276-
} while (0)
277-
#define COMP_FILE_CRC32(crc, data, len) \
278-
do { \
279-
if (use_crc32c) \
280-
COMP_CRC32C((crc), (data), (len)); \
281-
else \
282-
COMP_TRADITIONAL_CRC32(crc, data, len); \
283-
} while (0)
284-
#define FIN_FILE_CRC32(crc) \
285-
do { \
286-
if (use_crc32c) \
287-
FIN_CRC32C(crc); \
288-
else \
289-
FIN_TRADITIONAL_CRC32(crc); \
290-
} while (0)
291-
292270
/* open file in binary read mode */
293271
fp = fopen(file_path, PG_BINARY_R);
294272
if (fp == NULL)
295273
elog(ERROR, "cannot open file \"%s\": %s",
296274
file_path, strerror(errno));
297275

298276
/* calc CRC of backup file */
299-
INIT_FILE_CRC32(crc);
277+
INIT_FILE_CRC32(use_crc32c, crc);
300278
while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf))
301279
{
302280
if (interrupted)
303281
elog(ERROR, "interrupted during CRC calculation");
304-
COMP_FILE_CRC32(crc, buf, len);
282+
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
305283
}
306284
errno_tmp = errno;
307285
if (!feof(fp))
308286
elog(WARNING, "cannot read \"%s\": %s", file_path,
309287
strerror(errno_tmp));
310288
if (len > 0)
311-
COMP_FILE_CRC32(crc, buf, len);
312-
FIN_FILE_CRC32(crc);
289+
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
290+
FIN_FILE_CRC32(use_crc32c, crc);
313291

314292
fclose(fp);
315293

‎src/help.c‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ help_pg_probackup(void)
118118
printf(_(" [--master-db=db_name] [--master-host=host_name]\n"));
119119
printf(_(" [--master-port=port] [--master-user=user_name]\n"));
120120
printf(_(" [--replica-timeout=timeout]\n"));
121+
printf(_(" [--skip-block-validation]\n"));
121122

122123
printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
123124
printf(_(" [-D pgdata-path] [-i backup-id] [--progress]\n"));
@@ -127,12 +128,14 @@ help_pg_probackup(void)
127128
printf(_(" [--recovery-target-action=pause|promote|shutdown]\n"));
128129
printf(_(" [--restore-as-replica]\n"));
129130
printf(_(" [--no-validate]\n"));
131+
printf(_(" [--skip-block-validation]\n"));
130132

131133
printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME);
132134
printf(_(" [-i backup-id] [--progress]\n"));
133135
printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n"));
134136
printf(_(" [--recovery-target-name=target-name]\n"));
135137
printf(_(" [--timeline=timeline]\n"));
138+
printf(_(" [--skip-block-validation]\n"));
136139

137140
printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME);
138141
printf(_(" [--instance=instance_name [-i backup-id]]\n"));
@@ -203,7 +206,8 @@ help_backup(void)
203206
printf(_(" [-w --no-password] [-W --password]\n"));
204207
printf(_(" [--master-db=db_name] [--master-host=host_name]\n"));
205208
printf(_(" [--master-port=port] [--master-user=user_name]\n"));
206-
printf(_(" [--replica-timeout=timeout]\n\n"));
209+
printf(_(" [--replica-timeout=timeout]\n"));
210+
printf(_(" [--skip-block-validation]\n\n"));
207211

208212
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
209213
printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n"));
@@ -215,6 +219,7 @@ help_backup(void)
215219
printf(_(" -j, --threads=NUM number of parallel threads\n"));
216220
printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n"));
217221
printf(_(" --progress show progress\n"));
222+
printf(_(" --skip-block-validation set to validate only file-level checksum\n"));
218223

219224
printf(_("\n Logging options:\n"));
220225
printf(_(" --log-level-console=log-level-console\n"));
@@ -279,6 +284,7 @@ help_restore(void)
279284
printf(_(" [--immediate] [--recovery-target-name=target-name]\n"));
280285
printf(_(" [--recovery-target-action=pause|promote|shutdown]\n"));
281286
printf(_(" [--restore-as-replica] [--no-validate]\n\n"));
287+
printf(_(" [--skip-block-validation]\n\n"));
282288

283289
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
284290
printf(_(" --instance=instance_name name of the instance\n"));
@@ -305,6 +311,7 @@ help_restore(void)
305311
printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n"));
306312
printf(_(" to ease setting up a standby server\n"));
307313
printf(_(" --no-validate disable backup validation during restore\n"));
314+
printf(_(" --skip-block-validation set to validate only file-level checksum\n"));
308315

309316
printf(_("\n Logging options:\n"));
310317
printf(_(" --log-level-console=log-level-console\n"));
@@ -335,6 +342,7 @@ help_validate(void)
335342
printf(_(" [-i backup-id] [--progress]\n"));
336343
printf(_(" [--time=time|--xid=xid|--lsn=lsn [--inclusive=boolean]]\n"));
337344
printf(_(" [--timeline=timeline]\n\n"));
345+
printf(_(" [--skip-block-validation]\n\n"));
338346

339347
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
340348
printf(_(" --instance=instance_name name of the instance\n"));
@@ -348,6 +356,7 @@ help_validate(void)
348356
printf(_(" --timeline=timeline recovering into a particular timeline\n"));
349357
printf(_(" --recovery-target-name=target-name\n"));
350358
printf(_(" the named restore point to which recovery will proceed\n"));
359+
printf(_(" --skip-block-validation set to validate only file-level checksum\n"));
351360

352361
printf(_("\n Logging options:\n"));
353362
printf(_(" --log-level-console=log-level-console\n"));

‎src/pg_probackup.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@ static pgRecoveryTarget *recovery_target_options = NULL;
8989
bool restore_as_replica = false;
9090
bool restore_no_validate = false;
9191

92+
bool skip_block_validation = false;
93+
9294
/* delete options */
9395
bool delete_wal = false;
9496
bool delete_expired = false;
@@ -179,6 +181,7 @@ static pgut_option options[] =
179181
{ 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMDLINE },
180182
{ 'b', 27, "no-validate", &restore_no_validate, SOURCE_CMDLINE },
181183
{ 's', 28, "lsn", &target_lsn, SOURCE_CMDLINE },
184+
{ 'b', 29, "skip-block-validation", &skip_block_validation, SOURCE_CMDLINE },
182185
/* delete options */
183186
{ 'b', 130, "wal", &delete_wal, SOURCE_CMDLINE },
184187
{ 'b', 131, "expired", &delete_expired, SOURCE_CMDLINE },

‎src/pg_probackup.h‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,28 @@ typedef enum CompressAlg
6565
ZLIB_COMPRESS,
6666
} CompressAlg;
6767

68+
#define INIT_FILE_CRC32(use_crc32c, crc) \
69+
do { \
70+
if (use_crc32c) \
71+
INIT_CRC32C(crc); \
72+
else \
73+
INIT_TRADITIONAL_CRC32(crc); \
74+
} while (0)
75+
#define COMP_FILE_CRC32(use_crc32c, crc, data, len) \
76+
do { \
77+
if (use_crc32c) \
78+
COMP_CRC32C((crc), (data), (len)); \
79+
else \
80+
COMP_TRADITIONAL_CRC32(crc, data, len); \
81+
} while (0)
82+
#define FIN_FILE_CRC32(use_crc32c, crc) \
83+
do { \
84+
if (use_crc32c) \
85+
FIN_CRC32C(crc); \
86+
else \
87+
FIN_TRADITIONAL_CRC32(crc); \
88+
} while (0)
89+
6890
/* Information about single file (or dir) in backup */
6991
typedef struct pgFile
7092
{
@@ -339,6 +361,7 @@ extern bool exclusive_backup;
339361

340362
/* restore options */
341363
extern bool restore_as_replica;
364+
extern bool skip_block_validation;
342365

343366
/* delete options */
344367
extern bool delete_wal;
@@ -527,9 +550,9 @@ extern void get_wal_file(const char *from_path, const char *to_path);
527550

528551
extern bool calc_file_checksum(pgFile *file);
529552

530-
extern bool check_file_pages(pgFile* file, XLogRecPtr stop_lsn,
553+
extern bool check_file_pages(pgFile* file,
554+
XLogRecPtr stop_lsn,
531555
uint32 checksum_version, uint32 backup_version);
532-
533556
/* parsexlog.c */
534557
extern void extractPageMap(const char *archivedir,
535558
TimeLineID tli, uint32 seg_size,

‎src/validate.c‎

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -208,32 +208,42 @@ pgBackupValidateFiles(void *arg)
208208
}
209209

210210
/*
211-
* Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
212-
* use CRC-32.
213-
*
214-
* pg_control stores its content and checksum of the content, calculated
215-
* using CRC-32C. If we calculate checksum of the whole pg_control using
216-
* CRC-32C we get same checksum constantly. It might be because of the
217-
* CRC-32C algorithm.
218-
* To avoid this problem we need to use different algorithm, CRC-32 in
219-
* this case.
211+
* If option skip-block-validation is set, compute only file-level CRC for
212+
* datafiles, otherwise check them block by block.
220213
*/
221-
crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021);
222-
if (crc != file->crc)
214+
if (!file->is_datafile || skip_block_validation)
223215
{
224-
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
225-
file->path, file->crc, crc);
226-
arguments->corrupted = true;
227-
228-
/* validate relation blocks */
229-
if (file->is_datafile)
216+
/*
217+
* Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
218+
* use CRC-32.
219+
*
220+
* pg_control stores its content and checksum of the content, calculated
221+
* using CRC-32C. If we calculate checksum of the whole pg_control using
222+
* CRC-32C we get same checksum constantly. It might be because of the
223+
* CRC-32C algorithm.
224+
* To avoid this problem we need to use different algorithm, CRC-32 in
225+
* this case.
226+
*/
227+
crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021);
228+
if (crc != file->crc)
230229
{
231-
if (!check_file_pages(file, arguments->stop_lsn,
232-
arguments->checksum_version,
233-
arguments->backup_version))
234-
arguments->corrupted = true;
230+
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
231+
file->path, file->crc, crc);
232+
arguments->corrupted = true;
235233
}
236234
}
235+
else
236+
{
237+
/*
238+
* validate relation block by block
239+
* check page headers, checksums (if enabled)
240+
* and compute checksum of the file
241+
*/
242+
if (!check_file_pages(file, arguments->stop_lsn,
243+
arguments->checksum_version,
244+
arguments->backup_version))
245+
arguments->corrupted = true;
246+
}
237247
}
238248

239249
/* Data files validation is successful */

0 commit comments

Comments
 (0)