diff --git a/Makefile b/Makefile index f93cc37a..e449c9e5 100644 --- a/Makefile +++ b/Makefile @@ -87,3 +87,4 @@ endif include packaging/Makefile.pkg include packaging/Makefile.repo include packaging/Makefile.test + diff --git a/README.md b/README.md index 16aec1dc..da62364f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 13, 14, 15, 16, 17 +* PostgreSQL 13, 14, 15, 16, 17, 18 As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. @@ -79,7 +79,7 @@ For users of Postgres Pro products, commercial editions of pg_probackup are avai ## Building from source ### Linux -To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. Execute this in the module's directory: +To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. For versions under 18 execute this in the module's directory: ```shell make USE_PGXS=1 PG_CONFIG= top_srcdir= @@ -91,6 +91,8 @@ The alternative way, without using the PGXS infrastructure, is to place `pg_prob cd && git clone https://github.com/postgrespro/pg_probackup contrib/pg_probackup && cd contrib/pg_probackup && make ``` +For version 18 you have to apply PostgreSQL core patch (patches/REL_18_STABLE_pg_probackup.patch) first and recompile and reinstall PostgreSQL + ### Windows Currently pg_probackup can be build using only MSVC 2013. diff --git a/patches/REL_18_STABLE_pg_probackup.patch b/patches/REL_18_STABLE_pg_probackup.patch new file mode 100644 index 00000000..d47f0708 --- /dev/null +++ b/patches/REL_18_STABLE_pg_probackup.patch @@ -0,0 +1,29 @@ +From 5809673569d3d95c7bce75ea0aa6f9723e303c7d Mon Sep 17 00:00:00 2001 +From: Daria +Date: Mon, 17 Nov 2025 02:57:38 +0100 +Subject: [PATCH] REL_18_STABLE_pg_probackup + +--- + src/backend/utils/hash/pg_crc.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/backend/utils/hash/pg_crc.c b/src/backend/utils/hash/pg_crc.c +index e67a74ef852..6a474e804b5 100644 +--- a/src/backend/utils/hash/pg_crc.c ++++ b/src/backend/utils/hash/pg_crc.c +@@ -99,6 +99,7 @@ const uint32 pg_crc32_table[256] = { + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D + }; + ++#ifndef FRONTEND + /* + * SQL-callable functions + */ +@@ -128,3 +129,4 @@ crc32c_bytea(PG_FUNCTION_ARGS) + + PG_RETURN_INT64(crc); + } ++#endif +-- +2.39.5 (Apple Git-154) + diff --git a/src/backup.c b/src/backup.c index e1ebdb2c..3cbd4fbf 100644 --- a/src/backup.c +++ b/src/backup.c @@ -66,6 +66,7 @@ static bool pgpro_support(PGconn *conn); static bool pg_is_checksum_enabled(PGconn *conn); static bool pg_is_in_recovery(PGconn *conn); static bool pg_is_superuser(PGconn *conn); +static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); static void confirm_block_size(PGconn *conn, const char *name, int blcksz); static void rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); static bool remove_excluded_files_criterion(void *value, void *exclude_args); @@ -946,7 +947,7 @@ do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, /* * Confirm that this server version is supported */ -void +static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) { PGresult *res = NULL; @@ -2517,11 +2518,16 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) int segno; pgFile **file_item; pgFile f; +#if PG_VERSION_NUM >= 180000 + RelPathStr rel_path_str = relpathperm(rnode, forknum); + rel_path = rel_path_str.str; +#else + rel_path = relpathperm(rnode, forknum); +#endif segno = blkno / RELSEG_SIZE; blkno_inseg = blkno % RELSEG_SIZE; - rel_path = relpathperm(rnode, forknum); if (segno > 0) f.rel_path = psprintf("%s.%u", rel_path, segno); else @@ -2553,7 +2559,9 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) if (segno > 0) pg_free(f.rel_path); +#if PG_VERSION_NUM < 180000 pg_free(rel_path); +#endif } diff --git a/src/data.c b/src/data.c index d88b06b6..544adf18 100644 --- a/src/data.c +++ b/src/data.c @@ -1531,7 +1531,7 @@ validate_one_page(Page page, BlockNumber absolute_blkno, /* Verify checksum */ page_st->checksum = pg_checksum_page(page, absolute_blkno); - if (checksum_version) + if (checksum_version && !skip_block_validation) { /* Checksums are enabled, so check them. */ if (page_st->checksum != ((PageHeader) page)->pd_checksum) diff --git a/src/pg_probackup.c b/src/pg_probackup.c index b72efa93..030d64b0 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -687,7 +687,7 @@ main(int argc, char *argv[]) if (instance_config.pgdata != NULL && (backup_subcmd != ARCHIVE_GET_CMD && backup_subcmd != CATCHUP_CMD) && !is_absolute_path(instance_config.pgdata)) - elog(ERROR, "-D, --pgdata must be an absolute path"); + elog(ERROR, "-D, --pgdata must be an absolute path: %s", instance_config.pgdata); #if PG_VERSION_NUM >= 110000 /* Check xlog-seg-size option */ @@ -980,13 +980,7 @@ main(int argc, char *argv[]) wal_file_path, wal_file_name, batch_size, !no_validate_wal); break; case ADD_INSTANCE_CMD: - { - PGNodeInfo nodeInfo; - pgNodeInit(&nodeInfo); - instanceState->conn = pgut_connect(dbhost, dbport, dbname, dbuser); - check_server_version(instanceState->conn, &nodeInfo); - return do_add_instance(instanceState, &instance_config); - } + return do_add_instance(instanceState, &instance_config); case DELETE_INSTANCE_CMD: return do_delete_instance(instanceState); case INIT_CMD: diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 9ae66a3a..22a43a70 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1245,7 +1245,6 @@ extern const char *base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT extern long unsigned int base36dec(const char *text); extern uint32 parse_server_version(const char *server_version_str); extern uint32 parse_program_version(const char *program_version); -void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); extern bool parse_page(Page page, XLogRecPtr *lsn); extern int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, CompressAlg alg, int level, const char **errormsg); diff --git a/src/stream.c b/src/stream.c index 25429f9a..f136ed15 100644 --- a/src/stream.c +++ b/src/stream.c @@ -188,7 +188,11 @@ CreateReplicationSlot_compat(PGconn *conn, const char *slot_name, const char *pl bool is_temporary, bool is_physical, bool slot_exists_ok) { -#if PG_VERSION_NUM >= 150000 +#if PG_VERSION_NUM >= 180000 + return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, + /* reserve_wal = */ true, slot_exists_ok, /* two_phase = */ false, /* failover = */ false); +#elif PG_VERSION_NUM >= 150000 + return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, /* reserve_wal = */ true, slot_exists_ok, /* two_phase = */ false); #elif PG_VERSION_NUM >= 110000 diff --git a/src/util.c b/src/util.c index 019d2064..5189ba3b 100644 --- a/src/util.c +++ b/src/util.c @@ -93,8 +93,13 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) #endif if (size != ControlFileSize) + { + if (size == 16384) + elog(ERROR, "Unexpected control file size %d, expected %d. Probably you trying to connect Postgres Pro using %s built with PostgreSQL. ", + (int) size, ControlFileSize, PROGRAM_NAME); elog(ERROR, "Unexpected control file size %d, expected %d", (int) size, ControlFileSize); + } memcpy(ControlFile, src, sizeof(ControlFileData)); diff --git a/src/utils/pgut.c b/src/utils/pgut.c index df09dbdc..003f0a55 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -1062,7 +1062,23 @@ handle_interrupt(SIGNAL_ARGS) static void init_cancel_handler(void) { +#if PG_VERSION_NUM < 180000 oldhandler = pqsignal(SIGINT, handle_interrupt); +#else + { + struct sigaction act, oldact; + + act.sa_handler = handle_interrupt; + sigemptyset(&act.sa_mask); + act.sa_flags = SA_RESTART; + + /* Get the previous handler and set the new one */ + if (sigaction(SIGINT, &act, &oldact) < 0) + elog(ERROR, "sigaction(SIGINT) failed: %m"); + + oldhandler = oldact.sa_handler; + } +#endif pqsignal(SIGQUIT, handle_interrupt); pqsignal(SIGTERM, handle_interrupt); pqsignal(SIGPIPE, handle_interrupt); diff --git a/tests/archive_test.py b/tests/archive_test.py index 9bd37aa5..d2a141cb 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -2437,6 +2437,9 @@ def test_archive_get_prefetch_corruption(self): prefetch_line = 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename) restored_line = 'LOG: restored log file "{0}" from archive'.format(filename) + + self.wait_server_wal_exists(replica.data_dir, wal_dir, filename) + tailer = tail_file(os.path.join(replica.logs_dir, 'postgresql.log')) tailer.wait(contains=prefetch_line) tailer.wait(contains=restored_line) @@ -2483,6 +2486,9 @@ def test_archive_show_partial_files_handling(self): self.switch_wal_segment(node) + self.wait_instance_wal_exists(backup_dir, 'node', + f"{filename}.gz") + os.rename( os.path.join(wals_dir, filename), os.path.join(wals_dir, '{0}.part'.format(filename))) diff --git a/tests/auth_test.py b/tests/auth_test.py index 32cabc4a..eaef4e58 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -174,51 +174,36 @@ def test_backup_via_unprivileged_user(self): class AuthTest(unittest.TestCase): pb = None node = None + test_path = None - # TODO move to object scope, replace module_name - @classmethod - def setUpClass(cls): + @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") + def setUp(self): - super(AuthTest, cls).setUpClass() + super().setUp() - cls.pb = ProbackupTest() - cls.backup_dir = os.path.join(cls.pb.tmp_path, module_name, 'backup') + self.pb = ProbackupTest() + self.test_path = os.path.join(self.pb.tmp_path, module_name, self._testMethodName) + self.backup_dir = os.path.join(self.test_path, 'backup') - cls.node = cls.pb.make_simple_node( - base_dir="{}/node".format(module_name), + self.node = self.pb.make_simple_node( + base_dir=os.path.join(self.test_path, 'node'), set_replication=True, initdb_params=['--data-checksums', '--auth-host=md5'] ) - cls.username = cls.pb.get_username() + self.modify_pg_hba(self.node, self.pb.get_username()) - cls.modify_pg_hba(cls.node) - - cls.pb.init_pb(cls.backup_dir) - cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) - cls.pb.set_archiving(cls.backup_dir, cls.node.name, cls.node) + self.pb.init_pb(self.backup_dir) + self.pb.add_instance(self.backup_dir, self.node.name, self.node) + self.pb.set_archiving(self.backup_dir, self.node.name, self.node) try: - cls.node.slow_start() + self.node.slow_start() except StartNodeException: raise unittest.skip("Node hasn't started") - if cls.pb.get_version(cls.node) < 100000: - cls.node.safe_psql( - "postgres", - "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") - elif cls.pb.get_version(cls.node) < 150000: - cls.node.safe_psql( + version = self.pb.get_version(self.node) + if version < 150000: + self.node.safe_psql( "postgres", "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " @@ -233,7 +218,7 @@ def setUpClass(cls): "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") else: - cls.node.safe_psql( + self.node.safe_psql( "postgres", "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " @@ -247,16 +232,29 @@ def setUpClass(cls): "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") - cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') - - # TODO move to object scope, replace module_name - @classmethod - def tearDownClass(cls): - cls.node.cleanup() - cls.pb.del_test_dir(module_name, '') + if version >= 150000: + home_dir = os.path.join(self.test_path, "home") + os.makedirs(home_dir, exist_ok=True) + self.pb.test_env['HOME'] = home_dir + self.pgpass_file = os.path.join(home_dir, '.pgpass') + self.pgpass_file_lock = None + else: + # before PGv15 only true home dir were inspected. + # Since we can't have separate file per test, we have to serialize + # tests. + self.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') + self.pgpass_file_lock = self.pgpass_file + '~probackup_test_lock' + # have to lock pgpass by creating file in exclusive mode + for i in range(120): + try: + open(self.pgpass_file_lock, "x").close() + except FileExistsError: + time.sleep(1) + else: + break + else: + raise TimeoutError("can't create ~/.pgpass~probackup_test_lock for 120 seconds") - @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") - def setUp(self): self.pb_cmd = ['backup', '-B', self.backup_dir, '--instance', self.node.name, @@ -268,6 +266,19 @@ def setUp(self): ] def tearDown(self): + if (self.pgpass_file_lock + and hasattr(self, "pgpass_line") + and os.path.exists(self.pgpass_file)): + with open(self.pgpass_file, 'r', encoding="utf-8") as fl: + lines = fl.readlines() + if self.pgpass_line in lines: + lines.remove(self.pgpass_line) + if len(lines) == 0: + os.remove(self.pgpass_file) + else: + with open(self.pgpass_file, 'w', encoding="utf-8") as fl: + fl.writelines(lines) + if "PGPASSWORD" in self.pb.test_env.keys(): del self.pb.test_env["PGPASSWORD"] @@ -279,6 +290,10 @@ def tearDown(self): except OSError: pass + test_path = os.path.join(self.pb.tmp_path, module_name) + self.node.cleanup() + self.pb.del_test_dir(test_path, self._testMethodName) + def test_empty_password(self): """ Test case: PGPB_AUTH03 - zero password length """ try: @@ -313,7 +328,7 @@ def test_ctrl_c_event(self): def test_pgpassfile_env(self): """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """ - path = os.path.join(self.pb.tmp_path, module_name, 'pgpass.conf') + path = os.path.join(self.test_path, 'pgpass.conf') line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) self.create_pgpass(path, line) self.pb.test_env["PGPASSFILE"] = path @@ -367,7 +382,7 @@ def run_pb_with_auth(self, password=None, add_args = [], kill=False): @classmethod - def modify_pg_hba(cls, node): + def modify_pg_hba(self, node, username): """ Description: Add trust authentication for user postgres. Need for add new role and set grant. @@ -378,11 +393,12 @@ def modify_pg_hba(cls, node): with open(hba_conf, 'r+') as fio: data = fio.read() fio.seek(0) - fio.write('host\tall\t%s\t127.0.0.1/0\ttrust\n%s' % (cls.username, data)) + fio.write('host\tall\t%s\t127.0.0.1/0\ttrust\n%s' % (username, data)) def create_pgpass(self, path, line): + self.pgpass_line = line+"\n" with open(path, 'w') as passfile: # host:port:db:username:password - passfile.write(line) + passfile.write(self.pgpass_line) os.chmod(path, 0o600) diff --git a/tests/backup_test.py b/tests/backup_test.py index 5ec6f1c6..5e4e4fc3 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -3218,6 +3218,10 @@ def test_missing_replication_permission_1(self): replica.promote() + # Wait for replica to catch up with master before promoting + # to ensure 'backup' role is replicated + self.wait_until_replica_catch_with_master(node, replica) + # PAGE output = self.backup_node( backup_dir, 'node', replica, backup_type='page', diff --git a/tests/catchup_test.py b/tests/catchup_test.py index cf8388dd..2997a9c4 100644 --- a/tests/catchup_test.py +++ b/tests/catchup_test.py @@ -1586,18 +1586,22 @@ def test_dry_run_catchup_delta(self): # Cleanup src_pg.stop() + # Skip test, because it's PGDATA is global variable and has impact for other tests + # e.g. test_checkdb_amcheck_only_sanity + @unittest.skip("skip") def test_pgdata_is_ignored(self): """ In catchup we still allow PGDATA to be set either from command line or from the env var. This test that PGDATA is actually ignored and --source-pgadta is used instead """ - node = self.make_simple_node('node', + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication = True ) node.slow_start() # do full catchup - dest = self.make_empty_node('dst') + dest = self.make_empty_node(base_dir=os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = node.data_dir, @@ -1612,7 +1616,7 @@ def test_pgdata_is_ignored(self): os.environ['PGDATA']='xxx' - dest2 = self.make_empty_node('dst') + dest2 = self.make_empty_node(base_dir=os.path.join(self.module_name, self.fname, 'dst')) self.catchup_node( backup_mode = 'FULL', source_pgdata = node.data_dir, diff --git a/tests/checkdb_test.py b/tests/checkdb_test.py index eb46aea1..f5c0f689 100644 --- a/tests/checkdb_test.py +++ b/tests/checkdb_test.py @@ -18,7 +18,7 @@ def test_checkdb_amcheck_only_sanity(self): backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -227,7 +227,7 @@ def test_basic_checkdb_amcheck_only_sanity(self): """""" backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 2420be46..76f9ea3a 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -6,13 +6,14 @@ import signal import subprocess import shutil +from time import sleep import six import testgres import hashlib import re import getpass import select -from time import sleep +import time import re import json import random @@ -150,8 +151,9 @@ def __str__(self): class PostgresNodeExtended(testgres.PostgresNode): - def __init__(self, base_dir=None, *args, **kwargs): - super(PostgresNodeExtended, self).__init__(name='test', base_dir=base_dir, *args, **kwargs) + def __init__(self, base_dir=None, port = None, bin_dir=None, *args, **kwargs): + assert port is None or type(port) == int + super(PostgresNodeExtended, self).__init__(name='test', base_dir=base_dir, port = port, bin_dir=bin_dir, *args, **kwargs) self.is_started = False def slow_start(self, replica=False): @@ -414,25 +416,28 @@ def is_test_result_ok(test_case): # # 2. python versions 3.11+ mixin, verified on 3.11, taken from: https://stackoverflow.com/a/39606065 - if not isinstance(test_case, unittest.TestCase): - raise AssertionError("test_case is not instance of unittest.TestCase") - - if hasattr(test_case, '_outcome'): # Python 3.4+ - if hasattr(test_case._outcome, 'errors'): - # Python 3.4 - 3.10 (These two methods have no side effects) - result = test_case.defaultTestResult() # These two methods have no side effects - test_case._feedErrorsToResult(result, test_case._outcome.errors) - else: - # Python 3.11+ and pytest 5.3.5+ - result = test_case._outcome.result - if not hasattr(result, 'errors'): - result.errors = [] - if not hasattr(result, 'failures'): - result.failures = [] - else: # Python 2.7, 3.0-3.3 - result = getattr(test_case, '_outcomeForDoCleanups', test_case._resultForDoCleanups) + if hasattr(test_case._outcome, 'errors'): + # Python 3.4 - 3.10 (These two methods have no side effects) + result = test_case.defaultTestResult() # These two methods have no side effects + test_case._feedErrorsToResult(result, test_case._outcome.errors) + else: + # Python 3.11+ and pytest 5.3.5+ + result = test_case._outcome.result + if not hasattr(result, 'errors'): + result.errors = [] + if not hasattr(result, 'failures'): + result.failures = [] ok = all(test != test_case for test, text in result.errors + result.failures) + # check subtests as well + ok = ok and all(getattr(test, 'test_case', None) != test_case + for test, text in result.errors + result.failures) + + # for pytest 8+ + if hasattr(result, '_excinfo'): + if result._excinfo is not None and len(result._excinfo) > 0: + # if test was successful, _excinfo will be None, else it will be non-empty list + ok = False return ok @@ -475,12 +480,14 @@ def pg_config_version(self): def make_empty_node( self, - base_dir=None): + base_dir=None, + port=None, + bin_dir=None): real_base_dir = os.path.join(self.tmp_path, base_dir) shutil.rmtree(real_base_dir, ignore_errors=True) os.makedirs(real_base_dir) - node = PostgresNodeExtended(base_dir=real_base_dir) + node = PostgresNodeExtended(base_dir=real_base_dir, port=port, bin_dir=bin_dir) node.should_rm_dirs = True self.nodes_to_cleanup.append(node) @@ -489,12 +496,14 @@ def make_empty_node( def make_simple_node( self, base_dir=None, + port=None, + bin_dir=None, set_replication=False, ptrack_enable=False, initdb_params=[], pg_options={}): - node = self.make_empty_node(base_dir) + node = self.make_empty_node(base_dir, port=port, bin_dir=bin_dir) node.init( initdb_params=initdb_params, allow_streaming=set_replication) @@ -910,6 +919,24 @@ def get_backup_filelist_diff(self, filelist_A, filelist_B): return filelist_diff + def wait_instance_wal_exists(self, backup_dir, instance, file, timeout=300): + """Wait for WAL segment appeared in the WAL archive""" + start = time.time() + fl = f'wal/{instance}/{file}' + while time.time() - start < timeout: + if os.path.exists(fl): + break + time.sleep(0.25) + + def wait_server_wal_exists(self, data_dir, wal_dir, file, timeout=300): + """Wait for WAL segment appeared in the server WAL dir""" + start = time.time() + fl = f'{data_dir}/{wal_dir}/{file}' + while time.time() - start < timeout: + if os.path.exists(fl): + return + time.sleep(0.25) + # used for partial restore def truncate_every_file_in_dir(self, path): for file in os.listdir(path): diff --git a/tests/incr_restore_test.py b/tests/incr_restore_test.py index 6a216409..1c8f49e8 100644 --- a/tests/incr_restore_test.py +++ b/tests/incr_restore_test.py @@ -1592,17 +1592,12 @@ def test_incr_checksum_long_xact(self): 'select count(*) from t1').decode('utf-8').rstrip(), '1') - # @unittest.skip("skip") - # @unittest.expectedFailure - # This test will pass with Enterprise - # because it has checksums enabled by default - @unittest.skipIf(ProbackupTest.enterprise, 'skip') def test_incr_lsn_long_xact_1(self): """ """ node = self.make_simple_node( base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True) + set_replication=True, initdb_params=['--no-data-checksums']) backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) @@ -2046,8 +2041,9 @@ def test_incremental_partial_restore_exclude_lsn(self): base_dir=os.path.join(self.module_name, self.fname, 'node1')) node1.cleanup() - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2 = self.make_empty_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2'), port=node.port) + assert node2.port == node.port node2.cleanup() # restore some data into node2 @@ -2063,7 +2059,7 @@ def test_incremental_partial_restore_exclude_lsn(self): pgdata1 = self.pgdata_content(node1.data_dir) # partial incremental restore backup into node2 - node2.port = node.port + # node2.port = node.port node2.slow_start() node2.stop() self.restore_node( diff --git a/tests/ptrack_test.py b/tests/ptrack_test.py index 7b5bc416..38317ea2 100644 --- a/tests/ptrack_test.py +++ b/tests/ptrack_test.py @@ -1781,7 +1781,9 @@ def test_alter_database_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), + port = node.port) + assert node_restored.port == node.port node_restored.cleanup() self.restore_node( backup_dir, 'node', @@ -1799,7 +1801,6 @@ def test_alter_database_set_tablespace_ptrack(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - node_restored.port = node.port node_restored.slow_start() # @unittest.skip("skip") diff --git a/tests/requirements.txt b/tests/requirements.txt index 31e01aeb..0a0331b6 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,7 +5,7 @@ # git+https://github.com/postgrespro/testgres.git@ # 3. From a local directory # /path/to/local/directory/testgres -testgres==1.8.5 +testgres==1.12.0 allure-pytest deprecation pexpect diff --git a/tests/set_backup_test.py b/tests/set_backup_test.py index 31334cfb..cda29b7a 100644 --- a/tests/set_backup_test.py +++ b/tests/set_backup_test.py @@ -339,6 +339,9 @@ def test_wal_retention_and_pinning_1(self): node.pgbench_init(scale=2) + self.wait_instance_wal_exists(backup_dir, 'node', + "000000010000000000000004") + # Purge backups out = self.delete_expired( backup_dir, 'node', diff --git a/tests/validate_test.py b/tests/validate_test.py index 4ff44941..e8c6587f 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1711,7 +1711,7 @@ def test_validate_corrupt_wal_between_backups(self): wals_dir = os.path.join(backup_dir, 'wal', 'node') with open(os.path.join(wals_dir, walfile), "rb+", 0) as f: f.seek(9000) - f.write(b"b") + f.write(b"Because the answer is 42") f.flush() f.close @@ -1722,20 +1722,17 @@ def test_validate_corrupt_wal_between_backups(self): 'node', backup_id, options=[ - "--xid={0}".format(target_xid), "-j", "4"]) + f"--xid={target_xid}", "-j", "4"]) self.assertEqual( 1, 0, "Expecting Error because of wal segments corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) + f" Output: {repr(self.output)} \n CMD: {self.cmd}") except ProbackupException as e: self.assertTrue( 'ERROR: Not enough WAL records to xid' in e.message and 'WARNING: Recovery can be done up to time' in e.message and - "ERROR: Not enough WAL records to xid {0}\n".format( - target_xid), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + f"ERROR: Not enough WAL records to xid {target_xid}\n", + f'\n Unexpected Error Message: {repr(e.message)}\n CMD: {self.cmd}') self.assertEqual( 'OK', @@ -3397,6 +3394,10 @@ def test_corrupt_pg_control_via_resetxlog(self): os.path.join( backup_dir, 'backups', 'node', backup_id, 'database', wal_dir, 'archive_status')) + os.mkdir( + os.path.join( + backup_dir, 'backups', 'node', backup_id, 'database', wal_dir, 'summaries')) + pg_control_path = os.path.join( backup_dir, 'backups', 'node', backup_id, 'database', 'global', 'pg_control')